2024-11-23 14:24:56,414 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-23 14:24:56,431 main DEBUG Took 0.014088 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-23 14:24:56,431 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-23 14:24:56,431 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-23 14:24:56,433 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-23 14:24:56,434 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 14:24:56,445 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-23 14:24:56,466 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 14:24:56,467 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 14:24:56,468 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 14:24:56,469 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 14:24:56,470 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 14:24:56,470 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 14:24:56,471 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 14:24:56,472 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 14:24:56,473 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 14:24:56,473 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 14:24:56,475 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 14:24:56,475 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 14:24:56,480 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 14:24:56,481 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 14:24:56,482 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 14:24:56,482 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 14:24:56,483 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 14:24:56,483 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 14:24:56,484 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 14:24:56,485 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 14:24:56,485 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 14:24:56,487 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 14:24:56,488 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 14:24:56,489 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 14:24:56,490 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 14:24:56,490 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-23 14:24:56,492 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 14:24:56,494 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-23 14:24:56,497 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-23 14:24:56,498 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-23 14:24:56,499 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-23 14:24:56,500 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-23 14:24:56,508 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-23 14:24:56,510 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-23 14:24:56,513 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-23 14:24:56,513 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-23 14:24:56,514 main DEBUG createAppenders(={Console}) 2024-11-23 14:24:56,515 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-23 14:24:56,515 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-23 14:24:56,516 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-23 14:24:56,516 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-23 14:24:56,517 main DEBUG OutputStream closed 2024-11-23 14:24:56,517 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-23 14:24:56,517 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-23 14:24:56,518 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-23 14:24:56,602 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-23 14:24:56,605 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-23 14:24:56,606 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-23 14:24:56,606 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-23 14:24:56,607 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-23 14:24:56,607 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-23 14:24:56,608 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-23 14:24:56,608 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-23 14:24:56,609 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-23 14:24:56,609 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-23 14:24:56,609 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-23 14:24:56,610 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-23 14:24:56,610 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-23 14:24:56,611 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-23 14:24:56,611 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-23 14:24:56,611 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-23 14:24:56,612 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-23 14:24:56,613 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-23 14:24:56,614 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-23 14:24:56,615 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-23 14:24:56,615 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-23 14:24:56,616 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-23T14:24:56,866 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f90574b9-3de4-229a-1273-a648b15280c9 2024-11-23 14:24:56,869 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-23 14:24:56,869 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-23T14:24:56,879 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithNoInMemCompaction timeout: 13 mins 2024-11-23T14:24:56,905 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-23T14:24:56,927 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f90574b9-3de4-229a-1273-a648b15280c9/cluster_ea4d8525-e5f6-df4a-1eac-0a7a4e8ce8de, deleteOnExit=true 2024-11-23T14:24:56,928 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-23T14:24:56,929 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f90574b9-3de4-229a-1273-a648b15280c9/test.cache.data in system properties and HBase conf 2024-11-23T14:24:56,930 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f90574b9-3de4-229a-1273-a648b15280c9/hadoop.tmp.dir in system properties and HBase conf 2024-11-23T14:24:56,931 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f90574b9-3de4-229a-1273-a648b15280c9/hadoop.log.dir in system properties and HBase conf 2024-11-23T14:24:56,932 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f90574b9-3de4-229a-1273-a648b15280c9/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-23T14:24:56,933 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f90574b9-3de4-229a-1273-a648b15280c9/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-23T14:24:56,933 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-23T14:24:57,022 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-23T14:24:57,116 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-23T14:24:57,121 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f90574b9-3de4-229a-1273-a648b15280c9/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-23T14:24:57,122 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f90574b9-3de4-229a-1273-a648b15280c9/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-23T14:24:57,122 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f90574b9-3de4-229a-1273-a648b15280c9/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-23T14:24:57,123 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f90574b9-3de4-229a-1273-a648b15280c9/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-23T14:24:57,123 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f90574b9-3de4-229a-1273-a648b15280c9/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-23T14:24:57,124 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f90574b9-3de4-229a-1273-a648b15280c9/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-23T14:24:57,124 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f90574b9-3de4-229a-1273-a648b15280c9/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-23T14:24:57,125 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f90574b9-3de4-229a-1273-a648b15280c9/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-23T14:24:57,126 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f90574b9-3de4-229a-1273-a648b15280c9/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-23T14:24:57,126 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f90574b9-3de4-229a-1273-a648b15280c9/nfs.dump.dir in system properties and HBase conf 2024-11-23T14:24:57,127 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f90574b9-3de4-229a-1273-a648b15280c9/java.io.tmpdir in system properties and HBase conf 2024-11-23T14:24:57,127 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f90574b9-3de4-229a-1273-a648b15280c9/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-23T14:24:57,128 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f90574b9-3de4-229a-1273-a648b15280c9/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-23T14:24:57,129 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f90574b9-3de4-229a-1273-a648b15280c9/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-23T14:24:58,467 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-23T14:24:58,536 INFO [Time-limited test {}] log.Log(170): Logging initialized @2811ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-23T14:24:58,602 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-23T14:24:58,658 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-23T14:24:58,680 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-23T14:24:58,681 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-23T14:24:58,682 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-23T14:24:58,693 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-23T14:24:58,696 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7def033e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f90574b9-3de4-229a-1273-a648b15280c9/hadoop.log.dir/,AVAILABLE} 2024-11-23T14:24:58,697 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@41752735{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-23T14:24:58,873 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1bbf242f{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f90574b9-3de4-229a-1273-a648b15280c9/java.io.tmpdir/jetty-localhost-38193-hadoop-hdfs-3_4_1-tests_jar-_-any-7330823123001842338/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-23T14:24:58,880 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@34657887{HTTP/1.1, (http/1.1)}{localhost:38193} 2024-11-23T14:24:58,880 INFO [Time-limited test {}] server.Server(415): Started @3157ms 2024-11-23T14:24:59,451 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-23T14:24:59,459 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-23T14:24:59,460 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-23T14:24:59,460 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-23T14:24:59,461 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-23T14:24:59,462 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4b8e8cdd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f90574b9-3de4-229a-1273-a648b15280c9/hadoop.log.dir/,AVAILABLE} 2024-11-23T14:24:59,462 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5ef5f873{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-23T14:24:59,572 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3dfb1097{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f90574b9-3de4-229a-1273-a648b15280c9/java.io.tmpdir/jetty-localhost-39325-hadoop-hdfs-3_4_1-tests_jar-_-any-8539253428914390518/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-23T14:24:59,573 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@16142ff{HTTP/1.1, (http/1.1)}{localhost:39325} 2024-11-23T14:24:59,573 INFO [Time-limited test {}] server.Server(415): Started @3849ms 2024-11-23T14:24:59,628 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-23T14:25:00,734 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f90574b9-3de4-229a-1273-a648b15280c9/cluster_ea4d8525-e5f6-df4a-1eac-0a7a4e8ce8de/data/data1/current/BP-1838119457-172.17.0.3-1732371897891/current, will proceed with Du for space computation calculation, 2024-11-23T14:25:00,734 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f90574b9-3de4-229a-1273-a648b15280c9/cluster_ea4d8525-e5f6-df4a-1eac-0a7a4e8ce8de/data/data2/current/BP-1838119457-172.17.0.3-1732371897891/current, will proceed with Du for space computation calculation, 2024-11-23T14:25:00,760 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-23T14:25:00,799 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f90574b9-3de4-229a-1273-a648b15280c9 2024-11-23T14:25:00,804 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xef44c4776ffd6683 with lease ID 0x5aff43f924c044e3: Processing first storage report for DS-1ffad90b-d3ec-4937-92ac-11d25178b664 from datanode DatanodeRegistration(127.0.0.1:38519, datanodeUuid=d2a26ef9-0122-4224-9c19-f41d6fffe1e6, infoPort=44853, infoSecurePort=0, ipcPort=34983, storageInfo=lv=-57;cid=testClusterID;nsid=1114637223;c=1732371897891) 2024-11-23T14:25:00,805 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xef44c4776ffd6683 with lease ID 0x5aff43f924c044e3: from storage DS-1ffad90b-d3ec-4937-92ac-11d25178b664 node DatanodeRegistration(127.0.0.1:38519, datanodeUuid=d2a26ef9-0122-4224-9c19-f41d6fffe1e6, infoPort=44853, infoSecurePort=0, ipcPort=34983, storageInfo=lv=-57;cid=testClusterID;nsid=1114637223;c=1732371897891), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-23T14:25:00,806 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xef44c4776ffd6683 with lease ID 0x5aff43f924c044e3: Processing first storage report for DS-df3d3550-0caf-41ac-9ea6-2f3a666b09dd from datanode DatanodeRegistration(127.0.0.1:38519, datanodeUuid=d2a26ef9-0122-4224-9c19-f41d6fffe1e6, infoPort=44853, infoSecurePort=0, ipcPort=34983, storageInfo=lv=-57;cid=testClusterID;nsid=1114637223;c=1732371897891) 2024-11-23T14:25:00,806 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xef44c4776ffd6683 with lease ID 0x5aff43f924c044e3: from storage DS-df3d3550-0caf-41ac-9ea6-2f3a666b09dd node DatanodeRegistration(127.0.0.1:38519, datanodeUuid=d2a26ef9-0122-4224-9c19-f41d6fffe1e6, infoPort=44853, infoSecurePort=0, ipcPort=34983, storageInfo=lv=-57;cid=testClusterID;nsid=1114637223;c=1732371897891), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-23T14:25:00,861 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f90574b9-3de4-229a-1273-a648b15280c9/cluster_ea4d8525-e5f6-df4a-1eac-0a7a4e8ce8de/zookeeper_0, clientPort=50153, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f90574b9-3de4-229a-1273-a648b15280c9/cluster_ea4d8525-e5f6-df4a-1eac-0a7a4e8ce8de/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f90574b9-3de4-229a-1273-a648b15280c9/cluster_ea4d8525-e5f6-df4a-1eac-0a7a4e8ce8de/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-23T14:25:00,870 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=50153 2024-11-23T14:25:00,879 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T14:25:00,881 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T14:25:01,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741825_1001 (size=7) 2024-11-23T14:25:01,530 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a with version=8 2024-11-23T14:25:01,530 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/hbase-staging 2024-11-23T14:25:01,625 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-23T14:25:01,861 INFO [Time-limited test {}] client.ConnectionUtils(128): master/c6f7b1897123:0 server-side Connection retries=45 2024-11-23T14:25:01,870 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T14:25:01,871 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-23T14:25:01,875 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-23T14:25:01,875 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T14:25:01,876 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-23T14:25:02,004 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-23T14:25:02,054 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-23T14:25:02,062 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-23T14:25:02,065 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-23T14:25:02,092 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 32184 (auto-detected) 2024-11-23T14:25:02,093 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-11-23T14:25:02,110 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:38947 2024-11-23T14:25:02,130 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:38947 connecting to ZooKeeper ensemble=127.0.0.1:50153 2024-11-23T14:25:02,419 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:389470x0, quorum=127.0.0.1:50153, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-23T14:25:02,423 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:38947-0x101681565570000 connected 2024-11-23T14:25:02,645 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T14:25:02,649 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T14:25:02,661 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38947-0x101681565570000, quorum=127.0.0.1:50153, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T14:25:02,664 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a, hbase.cluster.distributed=false 2024-11-23T14:25:02,684 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38947-0x101681565570000, quorum=127.0.0.1:50153, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-23T14:25:02,688 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38947 2024-11-23T14:25:02,689 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38947 2024-11-23T14:25:02,689 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38947 2024-11-23T14:25:02,690 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38947 2024-11-23T14:25:02,690 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38947 2024-11-23T14:25:02,805 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/c6f7b1897123:0 server-side Connection retries=45 2024-11-23T14:25:02,807 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T14:25:02,807 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-23T14:25:02,807 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-23T14:25:02,807 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T14:25:02,808 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-23T14:25:02,810 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-23T14:25:02,813 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-23T14:25:02,814 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:35385 2024-11-23T14:25:02,817 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35385 connecting to ZooKeeper ensemble=127.0.0.1:50153 2024-11-23T14:25:02,819 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T14:25:02,822 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T14:25:02,838 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:353850x0, quorum=127.0.0.1:50153, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-23T14:25:02,839 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:353850x0, quorum=127.0.0.1:50153, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T14:25:02,839 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35385-0x101681565570001 connected 2024-11-23T14:25:02,843 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-23T14:25:02,850 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-23T14:25:02,852 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35385-0x101681565570001, quorum=127.0.0.1:50153, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-23T14:25:02,857 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35385-0x101681565570001, quorum=127.0.0.1:50153, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-23T14:25:02,858 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35385 2024-11-23T14:25:02,858 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35385 2024-11-23T14:25:02,859 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35385 2024-11-23T14:25:02,859 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35385 2024-11-23T14:25:02,860 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35385 2024-11-23T14:25:02,875 DEBUG [M:0;c6f7b1897123:38947 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;c6f7b1897123:38947 2024-11-23T14:25:02,876 INFO [master/c6f7b1897123:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/c6f7b1897123,38947,1732371901705 2024-11-23T14:25:02,892 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38947-0x101681565570000, quorum=127.0.0.1:50153, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T14:25:02,892 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35385-0x101681565570001, quorum=127.0.0.1:50153, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T14:25:02,893 DEBUG [master/c6f7b1897123:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38947-0x101681565570000, quorum=127.0.0.1:50153, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/c6f7b1897123,38947,1732371901705 2024-11-23T14:25:02,927 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35385-0x101681565570001, quorum=127.0.0.1:50153, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-23T14:25:02,927 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38947-0x101681565570000, quorum=127.0.0.1:50153, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T14:25:02,927 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35385-0x101681565570001, quorum=127.0.0.1:50153, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T14:25:02,928 DEBUG [master/c6f7b1897123:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38947-0x101681565570000, quorum=127.0.0.1:50153, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-23T14:25:02,929 INFO [master/c6f7b1897123:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/c6f7b1897123,38947,1732371901705 from backup master directory 2024-11-23T14:25:02,937 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35385-0x101681565570001, quorum=127.0.0.1:50153, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T14:25:02,937 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38947-0x101681565570000, quorum=127.0.0.1:50153, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/c6f7b1897123,38947,1732371901705 2024-11-23T14:25:02,938 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38947-0x101681565570000, quorum=127.0.0.1:50153, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T14:25:02,938 WARN [master/c6f7b1897123:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-23T14:25:02,938 INFO [master/c6f7b1897123:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=c6f7b1897123,38947,1732371901705 2024-11-23T14:25:02,941 INFO [master/c6f7b1897123:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-23T14:25:02,942 INFO [master/c6f7b1897123:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-23T14:25:02,998 DEBUG [master/c6f7b1897123:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/hbase.id] with ID: c88da961-458f-4375-b9d3-275f7f90019e 2024-11-23T14:25:02,998 DEBUG [master/c6f7b1897123:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/.tmp/hbase.id 2024-11-23T14:25:03,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741826_1002 (size=42) 2024-11-23T14:25:03,411 DEBUG [master/c6f7b1897123:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/.tmp/hbase.id]:[hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/hbase.id] 2024-11-23T14:25:03,458 INFO [master/c6f7b1897123:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T14:25:03,463 INFO [master/c6f7b1897123:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-23T14:25:03,482 INFO [master/c6f7b1897123:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms. 2024-11-23T14:25:03,537 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38947-0x101681565570000, quorum=127.0.0.1:50153, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T14:25:03,537 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35385-0x101681565570001, quorum=127.0.0.1:50153, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T14:25:03,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741827_1003 (size=196) 2024-11-23T14:25:03,976 INFO [master/c6f7b1897123:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-23T14:25:03,979 INFO [master/c6f7b1897123:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-23T14:25:03,993 DEBUG [master/c6f7b1897123:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:03,997 INFO [master/c6f7b1897123:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T14:25:04,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741828_1004 (size=1189) 2024-11-23T14:25:04,450 INFO [master/c6f7b1897123:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/MasterData/data/master/store 2024-11-23T14:25:04,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741829_1005 (size=34) 2024-11-23T14:25:04,880 INFO [master/c6f7b1897123:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-23T14:25:04,883 DEBUG [master/c6f7b1897123:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T14:25:04,884 DEBUG [master/c6f7b1897123:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-23T14:25:04,884 INFO [master/c6f7b1897123:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T14:25:04,884 DEBUG [master/c6f7b1897123:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T14:25:04,886 DEBUG [master/c6f7b1897123:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-23T14:25:04,886 DEBUG [master/c6f7b1897123:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T14:25:04,886 INFO [master/c6f7b1897123:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T14:25:04,887 DEBUG [master/c6f7b1897123:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732371904884Disabling compacts and flushes for region at 1732371904884Disabling writes for close at 1732371904886 (+2 ms)Writing region close event to WAL at 1732371904886Closed at 1732371904886 2024-11-23T14:25:04,890 WARN [master/c6f7b1897123:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/MasterData/data/master/store/.initializing 2024-11-23T14:25:04,890 DEBUG [master/c6f7b1897123:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/MasterData/WALs/c6f7b1897123,38947,1732371901705 2024-11-23T14:25:04,899 INFO [master/c6f7b1897123:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-23T14:25:04,914 INFO [master/c6f7b1897123:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c6f7b1897123%2C38947%2C1732371901705, suffix=, logDir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/MasterData/WALs/c6f7b1897123,38947,1732371901705, archiveDir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/MasterData/oldWALs, maxLogs=10 2024-11-23T14:25:04,935 DEBUG [master/c6f7b1897123:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/MasterData/WALs/c6f7b1897123,38947,1732371901705/c6f7b1897123%2C38947%2C1732371901705.1732371904918, exclude list is [], retry=0 2024-11-23T14:25:04,951 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38519,DS-1ffad90b-d3ec-4937-92ac-11d25178b664,DISK] 2024-11-23T14:25:04,954 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-23T14:25:04,988 INFO [master/c6f7b1897123:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/MasterData/WALs/c6f7b1897123,38947,1732371901705/c6f7b1897123%2C38947%2C1732371901705.1732371904918 2024-11-23T14:25:04,989 DEBUG [master/c6f7b1897123:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44853:44853)] 2024-11-23T14:25:04,990 DEBUG [master/c6f7b1897123:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-23T14:25:04,990 DEBUG [master/c6f7b1897123:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T14:25:04,994 DEBUG [master/c6f7b1897123:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T14:25:04,995 DEBUG [master/c6f7b1897123:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T14:25:05,030 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-23T14:25:05,053 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-23T14:25:05,057 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:05,060 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T14:25:05,061 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-23T14:25:05,065 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-23T14:25:05,065 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:05,066 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T14:25:05,066 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-23T14:25:05,069 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-23T14:25:05,070 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:05,071 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T14:25:05,071 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-23T14:25:05,074 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-23T14:25:05,074 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:05,075 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T14:25:05,076 DEBUG [master/c6f7b1897123:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T14:25:05,079 DEBUG [master/c6f7b1897123:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-23T14:25:05,081 DEBUG [master/c6f7b1897123:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-23T14:25:05,086 DEBUG [master/c6f7b1897123:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T14:25:05,086 DEBUG [master/c6f7b1897123:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T14:25:05,090 DEBUG [master/c6f7b1897123:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-23T14:25:05,093 DEBUG [master/c6f7b1897123:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T14:25:05,098 DEBUG [master/c6f7b1897123:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T14:25:05,099 INFO [master/c6f7b1897123:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63001407, jitterRate=-0.06120587885379791}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-23T14:25:05,105 DEBUG [master/c6f7b1897123:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732371905006Initializing all the Stores at 1732371905008 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732371905008Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732371905009 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732371905009Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732371905010 (+1 ms)Cleaning up temporary data from old regions at 1732371905086 (+76 ms)Region opened successfully at 1732371905105 (+19 ms) 2024-11-23T14:25:05,106 INFO [master/c6f7b1897123:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-23T14:25:05,136 DEBUG [master/c6f7b1897123:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9000e01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c6f7b1897123/172.17.0.3:0 2024-11-23T14:25:05,165 INFO [master/c6f7b1897123:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-23T14:25:05,175 INFO [master/c6f7b1897123:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-23T14:25:05,175 INFO [master/c6f7b1897123:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-23T14:25:05,177 INFO [master/c6f7b1897123:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-23T14:25:05,179 INFO [master/c6f7b1897123:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-23T14:25:05,183 INFO [master/c6f7b1897123:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-23T14:25:05,183 INFO [master/c6f7b1897123:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-23T14:25:05,207 INFO [master/c6f7b1897123:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-23T14:25:05,215 DEBUG [master/c6f7b1897123:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38947-0x101681565570000, quorum=127.0.0.1:50153, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-23T14:25:05,274 DEBUG [master/c6f7b1897123:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-23T14:25:05,277 INFO [master/c6f7b1897123:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-23T14:25:05,279 DEBUG [master/c6f7b1897123:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38947-0x101681565570000, quorum=127.0.0.1:50153, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-23T14:25:05,291 DEBUG [master/c6f7b1897123:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-23T14:25:05,294 INFO [master/c6f7b1897123:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-23T14:25:05,299 DEBUG [master/c6f7b1897123:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38947-0x101681565570000, quorum=127.0.0.1:50153, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-23T14:25:05,312 DEBUG [master/c6f7b1897123:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-23T14:25:05,315 DEBUG [master/c6f7b1897123:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38947-0x101681565570000, quorum=127.0.0.1:50153, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-23T14:25:05,327 DEBUG [master/c6f7b1897123:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-23T14:25:05,352 DEBUG [master/c6f7b1897123:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38947-0x101681565570000, quorum=127.0.0.1:50153, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-23T14:25:05,364 DEBUG [master/c6f7b1897123:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-23T14:25:05,379 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35385-0x101681565570001, quorum=127.0.0.1:50153, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-23T14:25:05,379 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38947-0x101681565570000, quorum=127.0.0.1:50153, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-23T14:25:05,379 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35385-0x101681565570001, quorum=127.0.0.1:50153, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T14:25:05,379 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38947-0x101681565570000, quorum=127.0.0.1:50153, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T14:25:05,382 INFO [master/c6f7b1897123:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=c6f7b1897123,38947,1732371901705, sessionid=0x101681565570000, setting cluster-up flag (Was=false) 2024-11-23T14:25:05,411 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38947-0x101681565570000, quorum=127.0.0.1:50153, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T14:25:05,411 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35385-0x101681565570001, quorum=127.0.0.1:50153, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T14:25:05,443 DEBUG [master/c6f7b1897123:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-23T14:25:05,446 DEBUG [master/c6f7b1897123:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=c6f7b1897123,38947,1732371901705 2024-11-23T14:25:05,470 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38947-0x101681565570000, quorum=127.0.0.1:50153, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T14:25:05,470 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35385-0x101681565570001, quorum=127.0.0.1:50153, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T14:25:05,513 DEBUG [master/c6f7b1897123:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-23T14:25:05,515 DEBUG [master/c6f7b1897123:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=c6f7b1897123,38947,1732371901705 2024-11-23T14:25:05,521 INFO [master/c6f7b1897123:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-23T14:25:05,566 INFO [RS:0;c6f7b1897123:35385 {}] regionserver.HRegionServer(746): ClusterId : c88da961-458f-4375-b9d3-275f7f90019e 2024-11-23T14:25:05,569 DEBUG [RS:0;c6f7b1897123:35385 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-23T14:25:05,581 DEBUG [RS:0;c6f7b1897123:35385 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-23T14:25:05,582 DEBUG [RS:0;c6f7b1897123:35385 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-23T14:25:05,591 DEBUG [RS:0;c6f7b1897123:35385 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-23T14:25:05,592 DEBUG [RS:0;c6f7b1897123:35385 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50a2ada5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c6f7b1897123/172.17.0.3:0 2024-11-23T14:25:05,591 DEBUG [master/c6f7b1897123:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-23T14:25:05,601 INFO [master/c6f7b1897123:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-23T14:25:05,607 DEBUG [RS:0;c6f7b1897123:35385 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;c6f7b1897123:35385 2024-11-23T14:25:05,608 INFO [master/c6f7b1897123:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-23T14:25:05,611 INFO [RS:0;c6f7b1897123:35385 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-23T14:25:05,611 INFO [RS:0;c6f7b1897123:35385 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-23T14:25:05,611 DEBUG [RS:0;c6f7b1897123:35385 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-23T14:25:05,614 INFO [RS:0;c6f7b1897123:35385 {}] regionserver.HRegionServer(2659): reportForDuty to master=c6f7b1897123,38947,1732371901705 with port=35385, startcode=1732371902767 2024-11-23T14:25:05,614 DEBUG [master/c6f7b1897123:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: c6f7b1897123,38947,1732371901705 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-23T14:25:05,621 DEBUG [master/c6f7b1897123:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/c6f7b1897123:0, corePoolSize=5, maxPoolSize=5 2024-11-23T14:25:05,621 DEBUG [master/c6f7b1897123:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/c6f7b1897123:0, corePoolSize=5, maxPoolSize=5 2024-11-23T14:25:05,621 DEBUG [master/c6f7b1897123:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/c6f7b1897123:0, corePoolSize=5, maxPoolSize=5 2024-11-23T14:25:05,621 DEBUG [master/c6f7b1897123:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/c6f7b1897123:0, corePoolSize=5, maxPoolSize=5 2024-11-23T14:25:05,621 DEBUG [master/c6f7b1897123:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/c6f7b1897123:0, corePoolSize=10, maxPoolSize=10 2024-11-23T14:25:05,622 DEBUG [master/c6f7b1897123:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/c6f7b1897123:0, corePoolSize=1, maxPoolSize=1 2024-11-23T14:25:05,622 DEBUG [master/c6f7b1897123:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/c6f7b1897123:0, corePoolSize=2, maxPoolSize=2 2024-11-23T14:25:05,622 DEBUG [master/c6f7b1897123:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/c6f7b1897123:0, corePoolSize=1, maxPoolSize=1 2024-11-23T14:25:05,624 INFO [master/c6f7b1897123:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732371935624 2024-11-23T14:25:05,626 INFO [master/c6f7b1897123:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-23T14:25:05,627 INFO [master/c6f7b1897123:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-23T14:25:05,627 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-23T14:25:05,628 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-23T14:25:05,628 DEBUG [RS:0;c6f7b1897123:35385 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-23T14:25:05,631 INFO [master/c6f7b1897123:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-23T14:25:05,632 INFO [master/c6f7b1897123:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-23T14:25:05,632 INFO [master/c6f7b1897123:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-23T14:25:05,632 INFO [master/c6f7b1897123:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-23T14:25:05,634 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:05,633 INFO [master/c6f7b1897123:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-23T14:25:05,635 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-23T14:25:05,636 INFO [master/c6f7b1897123:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-23T14:25:05,638 INFO [master/c6f7b1897123:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-23T14:25:05,638 INFO [master/c6f7b1897123:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-23T14:25:05,640 INFO [master/c6f7b1897123:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-23T14:25:05,641 INFO [master/c6f7b1897123:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-23T14:25:05,644 DEBUG [master/c6f7b1897123:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/c6f7b1897123:0:becomeActiveMaster-HFileCleaner.large.0-1732371905642,5,FailOnTimeoutGroup] 2024-11-23T14:25:05,645 DEBUG [master/c6f7b1897123:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/c6f7b1897123:0:becomeActiveMaster-HFileCleaner.small.0-1732371905644,5,FailOnTimeoutGroup] 2024-11-23T14:25:05,645 INFO [master/c6f7b1897123:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-23T14:25:05,645 INFO [master/c6f7b1897123:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-23T14:25:05,646 INFO [master/c6f7b1897123:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-23T14:25:05,647 INFO [master/c6f7b1897123:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-23T14:25:05,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741831_1007 (size=1321) 2024-11-23T14:25:05,668 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44127, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-23T14:25:05,675 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38947 {}] master.ServerManager(363): Checking decommissioned status of RegionServer c6f7b1897123,35385,1732371902767 2024-11-23T14:25:05,677 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38947 {}] master.ServerManager(517): Registering regionserver=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:05,693 DEBUG [RS:0;c6f7b1897123:35385 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a 2024-11-23T14:25:05,694 DEBUG [RS:0;c6f7b1897123:35385 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:43613 2024-11-23T14:25:05,694 DEBUG [RS:0;c6f7b1897123:35385 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-23T14:25:05,705 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38947-0x101681565570000, quorum=127.0.0.1:50153, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-23T14:25:05,706 DEBUG [RS:0;c6f7b1897123:35385 {}] zookeeper.ZKUtil(111): regionserver:35385-0x101681565570001, quorum=127.0.0.1:50153, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/c6f7b1897123,35385,1732371902767 2024-11-23T14:25:05,707 WARN [RS:0;c6f7b1897123:35385 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-23T14:25:05,707 INFO [RS:0;c6f7b1897123:35385 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T14:25:05,707 DEBUG [RS:0;c6f7b1897123:35385 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/WALs/c6f7b1897123,35385,1732371902767 2024-11-23T14:25:05,709 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [c6f7b1897123,35385,1732371902767] 2024-11-23T14:25:05,731 INFO [RS:0;c6f7b1897123:35385 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-23T14:25:05,747 INFO [RS:0;c6f7b1897123:35385 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-23T14:25:05,751 INFO [RS:0;c6f7b1897123:35385 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-23T14:25:05,752 INFO [RS:0;c6f7b1897123:35385 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T14:25:05,753 INFO [RS:0;c6f7b1897123:35385 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-23T14:25:05,759 INFO [RS:0;c6f7b1897123:35385 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-23T14:25:05,760 INFO [RS:0;c6f7b1897123:35385 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-23T14:25:05,761 DEBUG [RS:0;c6f7b1897123:35385 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/c6f7b1897123:0, corePoolSize=1, maxPoolSize=1 2024-11-23T14:25:05,761 DEBUG [RS:0;c6f7b1897123:35385 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/c6f7b1897123:0, corePoolSize=1, maxPoolSize=1 2024-11-23T14:25:05,761 DEBUG [RS:0;c6f7b1897123:35385 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/c6f7b1897123:0, corePoolSize=1, maxPoolSize=1 2024-11-23T14:25:05,762 DEBUG [RS:0;c6f7b1897123:35385 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/c6f7b1897123:0, corePoolSize=1, maxPoolSize=1 2024-11-23T14:25:05,762 DEBUG [RS:0;c6f7b1897123:35385 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/c6f7b1897123:0, corePoolSize=1, maxPoolSize=1 2024-11-23T14:25:05,762 DEBUG [RS:0;c6f7b1897123:35385 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/c6f7b1897123:0, corePoolSize=2, maxPoolSize=2 2024-11-23T14:25:05,762 DEBUG [RS:0;c6f7b1897123:35385 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/c6f7b1897123:0, corePoolSize=1, maxPoolSize=1 2024-11-23T14:25:05,763 DEBUG [RS:0;c6f7b1897123:35385 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/c6f7b1897123:0, corePoolSize=1, maxPoolSize=1 2024-11-23T14:25:05,763 DEBUG [RS:0;c6f7b1897123:35385 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/c6f7b1897123:0, corePoolSize=1, maxPoolSize=1 2024-11-23T14:25:05,763 DEBUG [RS:0;c6f7b1897123:35385 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/c6f7b1897123:0, corePoolSize=1, maxPoolSize=1 2024-11-23T14:25:05,763 DEBUG [RS:0;c6f7b1897123:35385 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/c6f7b1897123:0, corePoolSize=1, maxPoolSize=1 2024-11-23T14:25:05,764 DEBUG [RS:0;c6f7b1897123:35385 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/c6f7b1897123:0, corePoolSize=1, maxPoolSize=1 2024-11-23T14:25:05,764 DEBUG [RS:0;c6f7b1897123:35385 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/c6f7b1897123:0, corePoolSize=3, maxPoolSize=3 2024-11-23T14:25:05,764 DEBUG [RS:0;c6f7b1897123:35385 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0, corePoolSize=3, maxPoolSize=3 2024-11-23T14:25:05,770 INFO [RS:0;c6f7b1897123:35385 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-23T14:25:05,770 INFO [RS:0;c6f7b1897123:35385 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-23T14:25:05,770 INFO [RS:0;c6f7b1897123:35385 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T14:25:05,771 INFO [RS:0;c6f7b1897123:35385 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-23T14:25:05,771 INFO [RS:0;c6f7b1897123:35385 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-23T14:25:05,771 INFO [RS:0;c6f7b1897123:35385 {}] hbase.ChoreService(168): Chore ScheduledChore name=c6f7b1897123,35385,1732371902767-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-23T14:25:05,790 INFO [RS:0;c6f7b1897123:35385 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-23T14:25:05,792 INFO [RS:0;c6f7b1897123:35385 {}] hbase.ChoreService(168): Chore ScheduledChore name=c6f7b1897123,35385,1732371902767-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T14:25:05,792 INFO [RS:0;c6f7b1897123:35385 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T14:25:05,792 INFO [RS:0;c6f7b1897123:35385 {}] regionserver.Replication(171): c6f7b1897123,35385,1732371902767 started 2024-11-23T14:25:05,809 INFO [RS:0;c6f7b1897123:35385 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T14:25:05,810 INFO [RS:0;c6f7b1897123:35385 {}] regionserver.HRegionServer(1482): Serving as c6f7b1897123,35385,1732371902767, RpcServer on c6f7b1897123/172.17.0.3:35385, sessionid=0x101681565570001 2024-11-23T14:25:05,810 DEBUG [RS:0;c6f7b1897123:35385 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-23T14:25:05,811 DEBUG [RS:0;c6f7b1897123:35385 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager c6f7b1897123,35385,1732371902767 2024-11-23T14:25:05,811 DEBUG [RS:0;c6f7b1897123:35385 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c6f7b1897123,35385,1732371902767' 2024-11-23T14:25:05,811 DEBUG [RS:0;c6f7b1897123:35385 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-23T14:25:05,812 DEBUG [RS:0;c6f7b1897123:35385 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-23T14:25:05,813 DEBUG [RS:0;c6f7b1897123:35385 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-23T14:25:05,813 DEBUG [RS:0;c6f7b1897123:35385 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-23T14:25:05,813 DEBUG [RS:0;c6f7b1897123:35385 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager c6f7b1897123,35385,1732371902767 2024-11-23T14:25:05,813 DEBUG [RS:0;c6f7b1897123:35385 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c6f7b1897123,35385,1732371902767' 2024-11-23T14:25:05,813 DEBUG [RS:0;c6f7b1897123:35385 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-23T14:25:05,814 DEBUG [RS:0;c6f7b1897123:35385 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-23T14:25:05,815 DEBUG [RS:0;c6f7b1897123:35385 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-23T14:25:05,815 INFO [RS:0;c6f7b1897123:35385 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-23T14:25:05,815 INFO [RS:0;c6f7b1897123:35385 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-23T14:25:05,922 INFO [RS:0;c6f7b1897123:35385 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-23T14:25:05,927 INFO [RS:0;c6f7b1897123:35385 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c6f7b1897123%2C35385%2C1732371902767, suffix=, logDir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/WALs/c6f7b1897123,35385,1732371902767, archiveDir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/oldWALs, maxLogs=32 2024-11-23T14:25:05,945 DEBUG [RS:0;c6f7b1897123:35385 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/WALs/c6f7b1897123,35385,1732371902767/c6f7b1897123%2C35385%2C1732371902767.1732371905930, exclude list is [], retry=0 2024-11-23T14:25:05,951 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38519,DS-1ffad90b-d3ec-4937-92ac-11d25178b664,DISK] 2024-11-23T14:25:05,955 INFO [RS:0;c6f7b1897123:35385 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/WALs/c6f7b1897123,35385,1732371902767/c6f7b1897123%2C35385%2C1732371902767.1732371905930 2024-11-23T14:25:05,956 DEBUG [RS:0;c6f7b1897123:35385 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44853:44853)] 2024-11-23T14:25:06,054 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-23T14:25:06,054 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a 2024-11-23T14:25:06,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741833_1009 (size=32) 2024-11-23T14:25:06,469 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T14:25:06,472 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-23T14:25:06,476 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-23T14:25:06,476 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:06,477 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T14:25:06,477 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-23T14:25:06,481 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-23T14:25:06,481 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:06,482 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T14:25:06,482 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-23T14:25:06,485 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-23T14:25:06,485 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:06,486 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T14:25:06,486 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-23T14:25:06,488 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-23T14:25:06,488 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:06,489 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T14:25:06,490 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-23T14:25:06,491 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/hbase/meta/1588230740 2024-11-23T14:25:06,492 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/hbase/meta/1588230740 2024-11-23T14:25:06,494 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-23T14:25:06,494 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-23T14:25:06,495 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-23T14:25:06,498 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-23T14:25:06,501 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T14:25:06,502 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62158150, jitterRate=-0.0737713873386383}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-23T14:25:06,505 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732371906469Initializing all the Stores at 1732371906471 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732371906472 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732371906472Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732371906472Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732371906472Cleaning up temporary data from old regions at 1732371906494 (+22 ms)Region opened successfully at 1732371906505 (+11 ms) 2024-11-23T14:25:06,505 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-23T14:25:06,505 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-23T14:25:06,505 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-23T14:25:06,505 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-23T14:25:06,505 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-23T14:25:06,507 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-23T14:25:06,507 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732371906505Disabling compacts and flushes for region at 1732371906505Disabling writes for close at 1732371906505Writing region close event to WAL at 1732371906506 (+1 ms)Closed at 1732371906507 (+1 ms) 2024-11-23T14:25:06,510 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-23T14:25:06,510 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-23T14:25:06,516 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-23T14:25:06,523 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-23T14:25:06,527 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-23T14:25:06,685 DEBUG [c6f7b1897123:38947 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-23T14:25:06,697 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:06,703 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c6f7b1897123,35385,1732371902767, state=OPENING 2024-11-23T14:25:06,775 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-23T14:25:06,786 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38947-0x101681565570000, quorum=127.0.0.1:50153, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T14:25:06,786 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35385-0x101681565570001, quorum=127.0.0.1:50153, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T14:25:06,788 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T14:25:06,788 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T14:25:06,791 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-23T14:25:06,793 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=c6f7b1897123,35385,1732371902767}] 2024-11-23T14:25:06,974 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-23T14:25:06,977 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41717, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-23T14:25:06,988 INFO [RS_OPEN_META-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-23T14:25:06,988 INFO [RS_OPEN_META-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T14:25:06,989 INFO [RS_OPEN_META-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-23T14:25:06,992 INFO [RS_OPEN_META-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c6f7b1897123%2C35385%2C1732371902767.meta, suffix=.meta, logDir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/WALs/c6f7b1897123,35385,1732371902767, archiveDir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/oldWALs, maxLogs=32 2024-11-23T14:25:07,013 DEBUG [RS_OPEN_META-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/WALs/c6f7b1897123,35385,1732371902767/c6f7b1897123%2C35385%2C1732371902767.meta.1732371906995.meta, exclude list is [], retry=0 2024-11-23T14:25:07,017 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38519,DS-1ffad90b-d3ec-4937-92ac-11d25178b664,DISK] 2024-11-23T14:25:07,020 INFO [RS_OPEN_META-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/WALs/c6f7b1897123,35385,1732371902767/c6f7b1897123%2C35385%2C1732371902767.meta.1732371906995.meta 2024-11-23T14:25:07,021 DEBUG [RS_OPEN_META-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44853:44853)] 2024-11-23T14:25:07,021 DEBUG [RS_OPEN_META-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-23T14:25:07,023 DEBUG [RS_OPEN_META-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-23T14:25:07,027 DEBUG [RS_OPEN_META-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-23T14:25:07,032 INFO [RS_OPEN_META-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-23T14:25:07,037 DEBUG [RS_OPEN_META-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-23T14:25:07,038 DEBUG [RS_OPEN_META-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T14:25:07,039 DEBUG [RS_OPEN_META-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-23T14:25:07,039 DEBUG [RS_OPEN_META-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-23T14:25:07,042 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-23T14:25:07,043 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-23T14:25:07,044 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:07,044 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T14:25:07,045 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-23T14:25:07,046 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-23T14:25:07,046 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:07,047 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T14:25:07,047 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-23T14:25:07,049 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-23T14:25:07,049 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:07,050 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T14:25:07,050 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-23T14:25:07,052 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-23T14:25:07,052 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:07,053 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T14:25:07,053 DEBUG [RS_OPEN_META-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-23T14:25:07,054 DEBUG [RS_OPEN_META-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/hbase/meta/1588230740 2024-11-23T14:25:07,057 DEBUG [RS_OPEN_META-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/hbase/meta/1588230740 2024-11-23T14:25:07,059 DEBUG [RS_OPEN_META-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-23T14:25:07,059 DEBUG [RS_OPEN_META-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-23T14:25:07,060 DEBUG [RS_OPEN_META-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-23T14:25:07,063 DEBUG [RS_OPEN_META-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-23T14:25:07,065 INFO [RS_OPEN_META-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67541036, jitterRate=0.006439864635467529}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-23T14:25:07,065 DEBUG [RS_OPEN_META-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-23T14:25:07,067 DEBUG [RS_OPEN_META-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732371907039Writing region info on filesystem at 1732371907040 (+1 ms)Initializing all the Stores at 1732371907041 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732371907041Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732371907042 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732371907042Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732371907042Cleaning up temporary data from old regions at 1732371907060 (+18 ms)Running coprocessor post-open hooks at 1732371907065 (+5 ms)Region opened successfully at 1732371907067 (+2 ms) 2024-11-23T14:25:07,073 INFO [RS_OPEN_META-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732371906966 2024-11-23T14:25:07,083 DEBUG [RS_OPEN_META-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-23T14:25:07,084 INFO [RS_OPEN_META-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-23T14:25:07,085 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:07,088 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c6f7b1897123,35385,1732371902767, state=OPEN 2024-11-23T14:25:07,205 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38947-0x101681565570000, quorum=127.0.0.1:50153, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-23T14:25:07,205 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35385-0x101681565570001, quorum=127.0.0.1:50153, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-23T14:25:07,205 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T14:25:07,205 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T14:25:07,206 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:07,216 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-23T14:25:07,216 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=c6f7b1897123,35385,1732371902767 in 414 msec 2024-11-23T14:25:07,225 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-23T14:25:07,225 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 703 msec 2024-11-23T14:25:07,228 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-23T14:25:07,228 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-23T14:25:07,248 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T14:25:07,251 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6f7b1897123,35385,1732371902767, seqNum=-1] 2024-11-23T14:25:07,275 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T14:25:07,277 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58413, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T14:25:07,297 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.7460 sec 2024-11-23T14:25:07,297 INFO [master/c6f7b1897123:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732371907297, completionTime=-1 2024-11-23T14:25:07,300 INFO [master/c6f7b1897123:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-23T14:25:07,300 DEBUG [master/c6f7b1897123:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-23T14:25:07,324 INFO [master/c6f7b1897123:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-23T14:25:07,324 INFO [master/c6f7b1897123:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732371967324 2024-11-23T14:25:07,324 INFO [master/c6f7b1897123:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732372027324 2024-11-23T14:25:07,324 INFO [master/c6f7b1897123:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 23 msec 2024-11-23T14:25:07,326 INFO [master/c6f7b1897123:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c6f7b1897123,38947,1732371901705-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T14:25:07,326 INFO [master/c6f7b1897123:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c6f7b1897123,38947,1732371901705-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T14:25:07,326 INFO [master/c6f7b1897123:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c6f7b1897123,38947,1732371901705-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T14:25:07,328 INFO [master/c6f7b1897123:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-c6f7b1897123:38947, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T14:25:07,328 INFO [master/c6f7b1897123:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-23T14:25:07,328 INFO [master/c6f7b1897123:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-23T14:25:07,335 DEBUG [master/c6f7b1897123:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-23T14:25:07,359 INFO [master/c6f7b1897123:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 4.420sec 2024-11-23T14:25:07,360 INFO [master/c6f7b1897123:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-23T14:25:07,361 INFO [master/c6f7b1897123:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-23T14:25:07,362 INFO [master/c6f7b1897123:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-23T14:25:07,362 INFO [master/c6f7b1897123:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-23T14:25:07,362 INFO [master/c6f7b1897123:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-23T14:25:07,363 INFO [master/c6f7b1897123:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c6f7b1897123,38947,1732371901705-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-23T14:25:07,363 INFO [master/c6f7b1897123:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c6f7b1897123,38947,1732371901705-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-23T14:25:07,370 DEBUG [master/c6f7b1897123:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-23T14:25:07,371 INFO [master/c6f7b1897123:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-23T14:25:07,371 INFO [master/c6f7b1897123:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c6f7b1897123,38947,1732371901705-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T14:25:07,379 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e129cbd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:25:07,381 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-23T14:25:07,381 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-23T14:25:07,383 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6f7b1897123,38947,-1 for getting cluster id 2024-11-23T14:25:07,386 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T14:25:07,394 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c88da961-458f-4375-b9d3-275f7f90019e' 2024-11-23T14:25:07,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T14:25:07,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c88da961-458f-4375-b9d3-275f7f90019e" 2024-11-23T14:25:07,398 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@231891f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:25:07,398 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6f7b1897123,38947,-1] 2024-11-23T14:25:07,401 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T14:25:07,403 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:25:07,404 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58894, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T14:25:07,406 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ebea588, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:25:07,407 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T14:25:07,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6f7b1897123,35385,1732371902767, seqNum=-1] 2024-11-23T14:25:07,414 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T14:25:07,416 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34016, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T14:25:07,441 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=c6f7b1897123,38947,1732371901705 2024-11-23T14:25:07,453 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=426, ProcessCount=11, AvailableMemoryMB=3716 2024-11-23T14:25:07,462 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-23T14:25:07,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is c6f7b1897123,38947,1732371901705 2024-11-23T14:25:07,491 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@62ad2da3 2024-11-23T14:25:07,492 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-23T14:25:07,495 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58908, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-23T14:25:07,497 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-23T14:25:07,501 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-23T14:25:07,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-23T14:25:07,511 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-23T14:25:07,517 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-23T14:25:07,517 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:07,520 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-23T14:25:07,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-23T14:25:07,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741835_1011 (size=959) 2024-11-23T14:25:07,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-23T14:25:07,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-23T14:25:07,941 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a 2024-11-23T14:25:07,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741836_1012 (size=53) 2024-11-23T14:25:08,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-23T14:25:08,358 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T14:25:08,358 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 5d1382f36edf9c126fa17e1f99c7f3fe, disabling compactions & flushes 2024-11-23T14:25:08,359 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:08,359 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:08,359 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. after waiting 0 ms 2024-11-23T14:25:08,359 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:08,359 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:08,359 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 5d1382f36edf9c126fa17e1f99c7f3fe: Waiting for close lock at 1732371908358Disabling compacts and flushes for region at 1732371908358Disabling writes for close at 1732371908359 (+1 ms)Writing region close event to WAL at 1732371908359Closed at 1732371908359 2024-11-23T14:25:08,364 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-23T14:25:08,369 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732371908364"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732371908364"}]},"ts":"1732371908364"} 2024-11-23T14:25:08,374 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-23T14:25:08,375 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-23T14:25:08,378 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732371908375"}]},"ts":"1732371908375"} 2024-11-23T14:25:08,382 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-23T14:25:08,383 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5d1382f36edf9c126fa17e1f99c7f3fe, ASSIGN}] 2024-11-23T14:25:08,386 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5d1382f36edf9c126fa17e1f99c7f3fe, ASSIGN 2024-11-23T14:25:08,388 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5d1382f36edf9c126fa17e1f99c7f3fe, ASSIGN; state=OFFLINE, location=c6f7b1897123,35385,1732371902767; forceNewPlan=false, retain=false 2024-11-23T14:25:08,540 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=5d1382f36edf9c126fa17e1f99c7f3fe, regionState=OPENING, regionLocation=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:08,547 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5d1382f36edf9c126fa17e1f99c7f3fe, ASSIGN because future has completed 2024-11-23T14:25:08,549 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767}] 2024-11-23T14:25:08,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-23T14:25:08,713 INFO [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:08,713 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} 2024-11-23T14:25:08,714 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 5d1382f36edf9c126fa17e1f99c7f3fe 2024-11-23T14:25:08,714 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T14:25:08,714 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 5d1382f36edf9c126fa17e1f99c7f3fe 2024-11-23T14:25:08,714 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 5d1382f36edf9c126fa17e1f99c7f3fe 2024-11-23T14:25:08,716 INFO [StoreOpener-5d1382f36edf9c126fa17e1f99c7f3fe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 5d1382f36edf9c126fa17e1f99c7f3fe 2024-11-23T14:25:08,719 INFO [StoreOpener-5d1382f36edf9c126fa17e1f99c7f3fe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5d1382f36edf9c126fa17e1f99c7f3fe columnFamilyName A 2024-11-23T14:25:08,719 DEBUG [StoreOpener-5d1382f36edf9c126fa17e1f99c7f3fe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:08,720 INFO [StoreOpener-5d1382f36edf9c126fa17e1f99c7f3fe-1 {}] regionserver.HStore(327): Store=5d1382f36edf9c126fa17e1f99c7f3fe/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T14:25:08,720 INFO [StoreOpener-5d1382f36edf9c126fa17e1f99c7f3fe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 5d1382f36edf9c126fa17e1f99c7f3fe 2024-11-23T14:25:08,723 INFO [StoreOpener-5d1382f36edf9c126fa17e1f99c7f3fe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5d1382f36edf9c126fa17e1f99c7f3fe columnFamilyName B 2024-11-23T14:25:08,723 DEBUG [StoreOpener-5d1382f36edf9c126fa17e1f99c7f3fe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:08,724 INFO [StoreOpener-5d1382f36edf9c126fa17e1f99c7f3fe-1 {}] regionserver.HStore(327): Store=5d1382f36edf9c126fa17e1f99c7f3fe/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T14:25:08,724 INFO [StoreOpener-5d1382f36edf9c126fa17e1f99c7f3fe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 5d1382f36edf9c126fa17e1f99c7f3fe 2024-11-23T14:25:08,726 INFO [StoreOpener-5d1382f36edf9c126fa17e1f99c7f3fe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5d1382f36edf9c126fa17e1f99c7f3fe columnFamilyName C 2024-11-23T14:25:08,726 DEBUG [StoreOpener-5d1382f36edf9c126fa17e1f99c7f3fe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:08,727 INFO [StoreOpener-5d1382f36edf9c126fa17e1f99c7f3fe-1 {}] regionserver.HStore(327): Store=5d1382f36edf9c126fa17e1f99c7f3fe/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T14:25:08,727 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 5d1382f36edf9c126fa17e1f99c7f3fe 2024-11-23T14:25:08,729 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe 2024-11-23T14:25:08,730 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe 2024-11-23T14:25:08,731 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 5d1382f36edf9c126fa17e1f99c7f3fe 2024-11-23T14:25:08,732 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 5d1382f36edf9c126fa17e1f99c7f3fe 2024-11-23T14:25:08,732 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-23T14:25:08,735 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 5d1382f36edf9c126fa17e1f99c7f3fe 2024-11-23T14:25:08,738 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T14:25:08,739 INFO [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 5d1382f36edf9c126fa17e1f99c7f3fe; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63427431, jitterRate=-0.05485762655735016}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-23T14:25:08,739 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5d1382f36edf9c126fa17e1f99c7f3fe 2024-11-23T14:25:08,740 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 5d1382f36edf9c126fa17e1f99c7f3fe: Running coprocessor pre-open hook at 1732371908714Writing region info on filesystem at 1732371908714Initializing all the Stores at 1732371908716 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732371908716Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732371908716Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732371908716Cleaning up temporary data from old regions at 1732371908732 (+16 ms)Running coprocessor post-open hooks at 1732371908739 (+7 ms)Region opened successfully at 1732371908740 (+1 ms) 2024-11-23T14:25:08,742 INFO [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., pid=6, masterSystemTime=1732371908705 2024-11-23T14:25:08,745 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:08,746 INFO [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:08,747 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=5d1382f36edf9c126fa17e1f99c7f3fe, regionState=OPEN, openSeqNum=2, regionLocation=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:08,752 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 because future has completed 2024-11-23T14:25:08,761 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-23T14:25:08,763 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 in 207 msec 2024-11-23T14:25:08,767 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-23T14:25:08,767 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5d1382f36edf9c126fa17e1f99c7f3fe, ASSIGN in 379 msec 2024-11-23T14:25:08,769 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-23T14:25:08,769 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732371908769"}]},"ts":"1732371908769"} 2024-11-23T14:25:08,772 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-23T14:25:08,775 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-23T14:25:08,779 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2710 sec 2024-11-23T14:25:09,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-23T14:25:09,684 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-23T14:25:09,694 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16b3430e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:25:09,694 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6f7b1897123,38947,-1 for getting cluster id 2024-11-23T14:25:09,695 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T14:25:09,697 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c88da961-458f-4375-b9d3-275f7f90019e' 2024-11-23T14:25:09,698 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T14:25:09,698 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c88da961-458f-4375-b9d3-275f7f90019e" 2024-11-23T14:25:09,699 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bff7409, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:25:09,699 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6f7b1897123,38947,-1] 2024-11-23T14:25:09,699 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T14:25:09,699 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:25:09,701 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58926, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T14:25:09,703 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2538dc8d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:25:09,704 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T14:25:09,705 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6f7b1897123,35385,1732371902767, seqNum=-1] 2024-11-23T14:25:09,705 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T14:25:09,707 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34028, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T14:25:09,710 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-23T14:25:09,711 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is c6f7b1897123,38947,1732371901705 2024-11-23T14:25:09,712 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@481dbf66 2024-11-23T14:25:09,715 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-23T14:25:09,716 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58938, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-23T14:25:09,720 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7685e9fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:25:09,721 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6f7b1897123,38947,-1 for getting cluster id 2024-11-23T14:25:09,721 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T14:25:09,722 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c88da961-458f-4375-b9d3-275f7f90019e' 2024-11-23T14:25:09,722 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T14:25:09,723 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c88da961-458f-4375-b9d3-275f7f90019e" 2024-11-23T14:25:09,723 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@450047aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:25:09,723 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6f7b1897123,38947,-1] 2024-11-23T14:25:09,723 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T14:25:09,724 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:25:09,725 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58950, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T14:25:09,726 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5676fb0c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:25:09,727 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77e3eb48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:25:09,727 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6f7b1897123,38947,-1 for getting cluster id 2024-11-23T14:25:09,727 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T14:25:09,728 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c88da961-458f-4375-b9d3-275f7f90019e' 2024-11-23T14:25:09,729 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T14:25:09,729 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c88da961-458f-4375-b9d3-275f7f90019e" 2024-11-23T14:25:09,730 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@149abd79, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:25:09,730 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6f7b1897123,38947,-1] 2024-11-23T14:25:09,730 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T14:25:09,730 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:25:09,732 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58968, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T14:25:09,733 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a0960a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:25:09,734 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b4d8404, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:25:09,735 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6f7b1897123,38947,-1 for getting cluster id 2024-11-23T14:25:09,735 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T14:25:09,737 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c88da961-458f-4375-b9d3-275f7f90019e' 2024-11-23T14:25:09,737 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T14:25:09,737 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c88da961-458f-4375-b9d3-275f7f90019e" 2024-11-23T14:25:09,738 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@749fb54b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:25:09,738 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6f7b1897123,38947,-1] 2024-11-23T14:25:09,738 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T14:25:09,739 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:25:09,740 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58986, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T14:25:09,741 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4df7e3f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:25:09,743 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@382062a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:25:09,743 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6f7b1897123,38947,-1 for getting cluster id 2024-11-23T14:25:09,744 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T14:25:09,745 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c88da961-458f-4375-b9d3-275f7f90019e' 2024-11-23T14:25:09,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T14:25:09,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c88da961-458f-4375-b9d3-275f7f90019e" 2024-11-23T14:25:09,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f5367b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:25:09,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6f7b1897123,38947,-1] 2024-11-23T14:25:09,747 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T14:25:09,748 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:25:09,750 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59012, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T14:25:09,751 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68b3fb7c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:25:09,752 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e8246b4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:25:09,752 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6f7b1897123,38947,-1 for getting cluster id 2024-11-23T14:25:09,752 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T14:25:09,753 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c88da961-458f-4375-b9d3-275f7f90019e' 2024-11-23T14:25:09,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T14:25:09,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c88da961-458f-4375-b9d3-275f7f90019e" 2024-11-23T14:25:09,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bd84679, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:25:09,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6f7b1897123,38947,-1] 2024-11-23T14:25:09,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T14:25:09,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:25:09,756 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59026, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T14:25:09,757 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65a3b223, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:25:09,760 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2335953a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:25:09,760 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6f7b1897123,38947,-1 for getting cluster id 2024-11-23T14:25:09,761 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T14:25:09,762 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c88da961-458f-4375-b9d3-275f7f90019e' 2024-11-23T14:25:09,762 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T14:25:09,762 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c88da961-458f-4375-b9d3-275f7f90019e" 2024-11-23T14:25:09,763 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ef4082b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:25:09,763 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6f7b1897123,38947,-1] 2024-11-23T14:25:09,763 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T14:25:09,763 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:25:09,765 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59050, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T14:25:09,766 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fd5891c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:25:09,767 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29b7950b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:25:09,767 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6f7b1897123,38947,-1 for getting cluster id 2024-11-23T14:25:09,767 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T14:25:09,768 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c88da961-458f-4375-b9d3-275f7f90019e' 2024-11-23T14:25:09,769 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T14:25:09,769 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c88da961-458f-4375-b9d3-275f7f90019e" 2024-11-23T14:25:09,769 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64b8551d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:25:09,769 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6f7b1897123,38947,-1] 2024-11-23T14:25:09,769 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T14:25:09,770 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:25:09,771 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59064, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T14:25:09,772 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@517e68dd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:25:09,775 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65766813, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:25:09,775 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6f7b1897123,38947,-1 for getting cluster id 2024-11-23T14:25:09,775 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T14:25:09,777 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c88da961-458f-4375-b9d3-275f7f90019e' 2024-11-23T14:25:09,778 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T14:25:09,778 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c88da961-458f-4375-b9d3-275f7f90019e" 2024-11-23T14:25:09,778 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c1b5617, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:25:09,778 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6f7b1897123,38947,-1] 2024-11-23T14:25:09,779 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T14:25:09,779 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:25:09,780 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59088, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T14:25:09,782 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@425be669, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:25:09,785 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ec2e2c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:25:09,785 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6f7b1897123,38947,-1 for getting cluster id 2024-11-23T14:25:09,785 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T14:25:09,787 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c88da961-458f-4375-b9d3-275f7f90019e' 2024-11-23T14:25:09,787 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T14:25:09,788 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c88da961-458f-4375-b9d3-275f7f90019e" 2024-11-23T14:25:09,788 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@373a2bd7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:25:09,788 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6f7b1897123,38947,-1] 2024-11-23T14:25:09,789 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T14:25:09,789 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:25:09,790 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59096, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T14:25:09,792 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33abc70, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:25:09,797 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T14:25:09,797 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T14:25:09,797 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T14:25:09,800 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6f7b1897123,35385,1732371902767, seqNum=-1] 2024-11-23T14:25:09,800 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6f7b1897123,35385,1732371902767, seqNum=-1] 2024-11-23T14:25:09,800 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6f7b1897123,35385,1732371902767, seqNum=-1] 2024-11-23T14:25:09,800 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T14:25:09,800 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T14:25:09,800 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T14:25:09,800 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T14:25:09,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6f7b1897123,35385,1732371902767, seqNum=-1] 2024-11-23T14:25:09,802 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34034, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T14:25:09,802 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34046, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T14:25:09,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T14:25:09,803 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34032, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T14:25:09,805 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T14:25:09,805 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34062, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T14:25:09,807 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6f7b1897123,35385,1732371902767, seqNum=-1] 2024-11-23T14:25:09,807 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T14:25:09,807 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T14:25:09,809 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6f7b1897123,35385,1732371902767, seqNum=-1] 2024-11-23T14:25:09,809 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T14:25:09,809 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34072, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T14:25:09,811 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34082, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T14:25:09,812 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2] 2024-11-23T14:25:09,813 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T14:25:09,814 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2] 2024-11-23T14:25:09,815 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2] 2024-11-23T14:25:09,814 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2] 2024-11-23T14:25:09,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2] 2024-11-23T14:25:09,815 DEBUG [Thread-158 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6f7b1897123,35385,1732371902767, seqNum=-1] 2024-11-23T14:25:09,816 DEBUG [Thread-158 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T14:25:09,816 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2] 2024-11-23T14:25:09,818 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34090, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T14:25:09,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2] 2024-11-23T14:25:09,835 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T14:25:09,838 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T14:25:09,838 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6f7b1897123,35385,1732371902767, seqNum=-1] 2024-11-23T14:25:09,838 DEBUG [Thread-145 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T14:25:09,840 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6f7b1897123,35385,1732371902767, seqNum=-1] 2024-11-23T14:25:09,840 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T14:25:09,840 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34098, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T14:25:09,842 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34106, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T14:25:09,849 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2] 2024-11-23T14:25:09,854 DEBUG [Thread-145 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2] 2024-11-23T14:25:09,862 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-23T14:25:09,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-23T14:25:09,871 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T14:25:09,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-23T14:25:09,874 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T14:25:09,876 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T14:25:09,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on 5d1382f36edf9c126fa17e1f99c7f3fe 2024-11-23T14:25:09,920 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T14:25:09,925 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5d1382f36edf9c126fa17e1f99c7f3fe 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-23T14:25:09,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-23T14:25:10,035 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/2f3495313ddc4d57a58cc9f9f8057c51 is 50, key is test_row_0/A:col10/1732371909918/Put/seqid=0 2024-11-23T14:25:10,040 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-23T14:25:10,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:10,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. as already flushing 2024-11-23T14:25:10,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:10,062 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:10,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:10,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741837_1013 (size=16681) 2024-11-23T14:25:10,086 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/2f3495313ddc4d57a58cc9f9f8057c51 2024-11-23T14:25:10,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:10,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:10,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1732371970096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:10,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:10,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34034 deadline: 1732371970104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:10,121 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:10,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34106 deadline: 1732371970106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:10,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:10,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34082 deadline: 1732371970113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:10,128 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:10,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34062 deadline: 1732371970121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:10,145 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:10,145 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:10,145 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:10,145 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:10,145 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:10,146 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:10,146 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:10,146 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:10,146 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:10,146 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:10,147 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:10,147 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:10,147 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:10,147 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:10,147 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:10,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-23T14:25:10,234 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/59f43f9d881248d5891b77dd7aa0a3ed is 50, key is test_row_0/B:col10/1732371909918/Put/seqid=0 2024-11-23T14:25:10,249 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-23T14:25:10,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:10,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1732371970251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:10,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:10,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:10,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:10,259 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:10,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34082 deadline: 1732371970253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:10,261 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:10,261 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:10,261 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:10,261 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:10,261 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:10,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34106 deadline: 1732371970253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:10,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34062 deadline: 1732371970254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:10,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:10,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:10,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:10,264 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:10,264 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:10,264 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:10,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:10,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34034 deadline: 1732371970254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:10,268 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:10,268 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:10,268 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:10,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:10,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. as already flushing 2024-11-23T14:25:10,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:10,277 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:10,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:10,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:10,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741838_1014 (size=12001) 2024-11-23T14:25:10,435 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-23T14:25:10,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:10,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. as already flushing 2024-11-23T14:25:10,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:10,437 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:10,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:10,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:10,475 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:10,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1732371970471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:10,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:10,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:10,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:10,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:10,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34106 deadline: 1732371970471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:10,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:10,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34082 deadline: 1732371970472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:10,477 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:10,477 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:10,477 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:10,477 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:10,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34062 deadline: 1732371970472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:10,478 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:10,478 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:10,478 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:10,481 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:10,482 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:10,482 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:10,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:10,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34034 deadline: 1732371970481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:10,483 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:10,483 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:10,483 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:10,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-23T14:25:10,595 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-23T14:25:10,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:10,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. as already flushing 2024-11-23T14:25:10,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:10,596 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:10,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:10,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:10,689 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/59f43f9d881248d5891b77dd7aa0a3ed 2024-11-23T14:25:10,740 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/00f2e7c3be6f4d0d914874f9671f60bf is 50, key is test_row_0/C:col10/1732371909918/Put/seqid=0 2024-11-23T14:25:10,753 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-23T14:25:10,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:10,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. as already flushing 2024-11-23T14:25:10,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:10,754 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:10,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:10,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:10,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741839_1015 (size=12001) 2024-11-23T14:25:10,770 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/00f2e7c3be6f4d0d914874f9671f60bf 2024-11-23T14:25:10,785 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:10,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1732371970781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:10,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:10,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:10,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:10,786 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:10,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34082 deadline: 1732371970781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:10,787 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:10,787 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:10,787 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:10,793 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/2f3495313ddc4d57a58cc9f9f8057c51 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/2f3495313ddc4d57a58cc9f9f8057c51 2024-11-23T14:25:10,793 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:10,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34106 deadline: 1732371970791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:10,795 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:10,795 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:10,795 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:10,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:10,797 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:10,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34034 deadline: 1732371970792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:10,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34062 deadline: 1732371970791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:10,798 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:10,798 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:10,798 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:10,798 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:10,798 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:10,798 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:10,815 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/2f3495313ddc4d57a58cc9f9f8057c51, entries=250, sequenceid=13, filesize=16.3 K 2024-11-23T14:25:10,822 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/59f43f9d881248d5891b77dd7aa0a3ed as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/59f43f9d881248d5891b77dd7aa0a3ed 2024-11-23T14:25:10,836 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/59f43f9d881248d5891b77dd7aa0a3ed, entries=150, sequenceid=13, filesize=11.7 K 2024-11-23T14:25:10,840 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/00f2e7c3be6f4d0d914874f9671f60bf as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/00f2e7c3be6f4d0d914874f9671f60bf 2024-11-23T14:25:10,855 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/00f2e7c3be6f4d0d914874f9671f60bf, entries=150, sequenceid=13, filesize=11.7 K 2024-11-23T14:25:10,866 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 5d1382f36edf9c126fa17e1f99c7f3fe in 937ms, sequenceid=13, compaction requested=false 2024-11-23T14:25:10,867 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-23T14:25:10,870 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5d1382f36edf9c126fa17e1f99c7f3fe: 2024-11-23T14:25:10,910 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-23T14:25:10,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:10,911 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 5d1382f36edf9c126fa17e1f99c7f3fe 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-23T14:25:10,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/be37907be43e42c4a5cc28e8ade494a8 is 50, key is test_row_0/A:col10/1732371910104/Put/seqid=0 2024-11-23T14:25:10,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741840_1016 (size=12001) 2024-11-23T14:25:10,975 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/be37907be43e42c4a5cc28e8ade494a8 2024-11-23T14:25:11,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-23T14:25:11,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/a804626854494ff782430763fdc6e74d is 50, key is test_row_0/B:col10/1732371910104/Put/seqid=0 2024-11-23T14:25:11,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741841_1017 (size=12001) 2024-11-23T14:25:11,055 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/a804626854494ff782430763fdc6e74d 2024-11-23T14:25:11,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/dbd269da3b0b42d5891afedba4052f31 is 50, key is test_row_0/C:col10/1732371910104/Put/seqid=0 2024-11-23T14:25:11,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741842_1018 (size=12001) 2024-11-23T14:25:11,128 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/dbd269da3b0b42d5891afedba4052f31 2024-11-23T14:25:11,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/be37907be43e42c4a5cc28e8ade494a8 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/be37907be43e42c4a5cc28e8ade494a8 2024-11-23T14:25:11,161 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/be37907be43e42c4a5cc28e8ade494a8, entries=150, sequenceid=38, filesize=11.7 K 2024-11-23T14:25:11,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/a804626854494ff782430763fdc6e74d as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/a804626854494ff782430763fdc6e74d 2024-11-23T14:25:11,179 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/a804626854494ff782430763fdc6e74d, entries=150, sequenceid=38, filesize=11.7 K 2024-11-23T14:25:11,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/dbd269da3b0b42d5891afedba4052f31 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/dbd269da3b0b42d5891afedba4052f31 2024-11-23T14:25:11,199 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/dbd269da3b0b42d5891afedba4052f31, entries=150, sequenceid=38, filesize=11.7 K 2024-11-23T14:25:11,201 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=0 B/0 for 5d1382f36edf9c126fa17e1f99c7f3fe in 290ms, sequenceid=38, compaction requested=false 2024-11-23T14:25:11,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 5d1382f36edf9c126fa17e1f99c7f3fe: 2024-11-23T14:25:11,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:11,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-23T14:25:11,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-23T14:25:11,211 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-23T14:25:11,212 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3280 sec 2024-11-23T14:25:11,219 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 1.3500 sec 2024-11-23T14:25:11,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on 5d1382f36edf9c126fa17e1f99c7f3fe 2024-11-23T14:25:11,330 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T14:25:11,330 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5d1382f36edf9c126fa17e1f99c7f3fe 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-23T14:25:11,346 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/36a49bff398744afa17989f1918bc895 is 50, key is test_row_0/A:col10/1732371911326/Put/seqid=0 2024-11-23T14:25:11,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741843_1019 (size=16677) 2024-11-23T14:25:11,390 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/36a49bff398744afa17989f1918bc895 2024-11-23T14:25:11,414 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/454815fae1124de5970090810f0634f0 is 50, key is test_row_0/B:col10/1732371911326/Put/seqid=0 2024-11-23T14:25:11,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:11,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34034 deadline: 1732371971414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:11,435 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:11,435 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:11,435 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:11,436 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:11,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34106 deadline: 1732371971417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:11,438 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:11,438 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:11,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741844_1020 (size=9657) 2024-11-23T14:25:11,438 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:11,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:11,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:11,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34082 deadline: 1732371971422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:11,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1732371971420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:11,439 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/454815fae1124de5970090810f0634f0 2024-11-23T14:25:11,439 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:11,439 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:11,439 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:11,439 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:11,439 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:11,439 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:11,440 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:11,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34062 deadline: 1732371971428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:11,441 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:11,441 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:11,441 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:11,461 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/bde64bd3db344bffbf0eb6c6a52fcdb9 is 50, key is test_row_0/C:col10/1732371911326/Put/seqid=0 2024-11-23T14:25:11,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741845_1021 (size=9657) 2024-11-23T14:25:11,481 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/bde64bd3db344bffbf0eb6c6a52fcdb9 2024-11-23T14:25:11,498 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/36a49bff398744afa17989f1918bc895 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/36a49bff398744afa17989f1918bc895 2024-11-23T14:25:11,514 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/36a49bff398744afa17989f1918bc895, entries=250, sequenceid=49, filesize=16.3 K 2024-11-23T14:25:11,518 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/454815fae1124de5970090810f0634f0 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/454815fae1124de5970090810f0634f0 2024-11-23T14:25:11,544 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/454815fae1124de5970090810f0634f0, entries=100, sequenceid=49, filesize=9.4 K 2024-11-23T14:25:11,546 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:11,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34034 deadline: 1732371971541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:11,547 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:11,547 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:11,548 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:11,549 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:11,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34106 deadline: 1732371971542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:11,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:11,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:11,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:11,552 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/bde64bd3db344bffbf0eb6c6a52fcdb9 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/bde64bd3db344bffbf0eb6c6a52fcdb9 2024-11-23T14:25:11,558 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:11,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34082 deadline: 1732371971551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:11,559 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:11,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:11,559 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:11,559 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:11,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1732371971551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:11,560 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:11,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34062 deadline: 1732371971551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:11,560 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:11,560 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:11,560 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:11,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:11,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:11,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:11,572 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/bde64bd3db344bffbf0eb6c6a52fcdb9, entries=100, sequenceid=49, filesize=9.4 K 2024-11-23T14:25:11,575 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 5d1382f36edf9c126fa17e1f99c7f3fe in 245ms, sequenceid=49, compaction requested=true 2024-11-23T14:25:11,575 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5d1382f36edf9c126fa17e1f99c7f3fe: 2024-11-23T14:25:11,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d1382f36edf9c126fa17e1f99c7f3fe:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T14:25:11,591 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:25:11,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:25:11,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d1382f36edf9c126fa17e1f99c7f3fe:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T14:25:11,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:25:11,594 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:25:11,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d1382f36edf9c126fa17e1f99c7f3fe:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T14:25:11,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:25:11,597 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:25:11,599 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1541): 5d1382f36edf9c126fa17e1f99c7f3fe/B is initiating minor compaction (all files) 2024-11-23T14:25:11,600 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d1382f36edf9c126fa17e1f99c7f3fe/B in TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:11,600 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/59f43f9d881248d5891b77dd7aa0a3ed, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/a804626854494ff782430763fdc6e74d, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/454815fae1124de5970090810f0634f0] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp, totalSize=32.9 K 2024-11-23T14:25:11,605 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 59f43f9d881248d5891b77dd7aa0a3ed, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732371909877 2024-11-23T14:25:11,607 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting a804626854494ff782430763fdc6e74d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1732371910058 2024-11-23T14:25:11,608 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 454815fae1124de5970090810f0634f0, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732371911324 2024-11-23T14:25:11,620 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 45359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:25:11,621 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1541): 5d1382f36edf9c126fa17e1f99c7f3fe/A is initiating minor compaction (all files) 2024-11-23T14:25:11,621 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d1382f36edf9c126fa17e1f99c7f3fe/A in TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:11,621 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/2f3495313ddc4d57a58cc9f9f8057c51, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/be37907be43e42c4a5cc28e8ade494a8, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/36a49bff398744afa17989f1918bc895] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp, totalSize=44.3 K 2024-11-23T14:25:11,625 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f3495313ddc4d57a58cc9f9f8057c51, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732371909877 2024-11-23T14:25:11,626 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting be37907be43e42c4a5cc28e8ade494a8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1732371910058 2024-11-23T14:25:11,630 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 36a49bff398744afa17989f1918bc895, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732371911307 2024-11-23T14:25:11,653 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d1382f36edf9c126fa17e1f99c7f3fe#B#compaction#9 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:25:11,655 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/e2ed9146f9c644d197142d726542a25e is 50, key is test_row_0/B:col10/1732371911326/Put/seqid=0 2024-11-23T14:25:11,681 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d1382f36edf9c126fa17e1f99c7f3fe#A#compaction#10 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:25:11,683 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/130a7944e2004a1a8cc64d56b2a61d9f is 50, key is test_row_0/A:col10/1732371911326/Put/seqid=0 2024-11-23T14:25:11,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741846_1022 (size=12104) 2024-11-23T14:25:11,727 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-23T14:25:11,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741847_1023 (size=12104) 2024-11-23T14:25:11,759 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/130a7944e2004a1a8cc64d56b2a61d9f as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/130a7944e2004a1a8cc64d56b2a61d9f 2024-11-23T14:25:11,765 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T14:25:11,766 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5d1382f36edf9c126fa17e1f99c7f3fe 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-23T14:25:11,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on 5d1382f36edf9c126fa17e1f99c7f3fe 2024-11-23T14:25:11,779 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/062b372f06be4727bc2b62598dbbbfcb is 50, key is test_row_0/A:col10/1732371911373/Put/seqid=0 2024-11-23T14:25:11,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741848_1024 (size=14341) 2024-11-23T14:25:11,800 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/062b372f06be4727bc2b62598dbbbfcb 2024-11-23T14:25:11,800 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5d1382f36edf9c126fa17e1f99c7f3fe/A of 5d1382f36edf9c126fa17e1f99c7f3fe into 130a7944e2004a1a8cc64d56b2a61d9f(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:25:11,801 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d1382f36edf9c126fa17e1f99c7f3fe: 2024-11-23T14:25:11,804 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., storeName=5d1382f36edf9c126fa17e1f99c7f3fe/A, priority=13, startTime=1732371911577; duration=0sec 2024-11-23T14:25:11,805 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:25:11,805 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d1382f36edf9c126fa17e1f99c7f3fe:A 2024-11-23T14:25:11,805 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:25:11,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:11,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34082 deadline: 1732371971789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:11,809 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:11,809 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:11,809 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:11,812 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:25:11,812 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1541): 5d1382f36edf9c126fa17e1f99c7f3fe/C is initiating minor compaction (all files) 2024-11-23T14:25:11,812 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d1382f36edf9c126fa17e1f99c7f3fe/C in TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:11,812 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/00f2e7c3be6f4d0d914874f9671f60bf, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/dbd269da3b0b42d5891afedba4052f31, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/bde64bd3db344bffbf0eb6c6a52fcdb9] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp, totalSize=32.9 K 2024-11-23T14:25:11,815 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 00f2e7c3be6f4d0d914874f9671f60bf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732371909877 2024-11-23T14:25:11,816 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting dbd269da3b0b42d5891afedba4052f31, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1732371910058 2024-11-23T14:25:11,817 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting bde64bd3db344bffbf0eb6c6a52fcdb9, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732371911324 2024-11-23T14:25:11,821 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:11,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1732371971800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:11,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:11,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:11,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:11,830 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:11,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34034 deadline: 1732371971807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:11,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:11,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34062 deadline: 1732371971805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:11,832 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:11,832 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:11,832 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:11,832 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:11,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34106 deadline: 1732371971807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:11,833 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:11,833 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:11,833 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:11,835 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:11,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:11,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:11,844 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/e8caa35907ed427fa90bea148eb97606 is 50, key is test_row_0/B:col10/1732371911373/Put/seqid=0 2024-11-23T14:25:11,852 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d1382f36edf9c126fa17e1f99c7f3fe#C#compaction#13 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:25:11,853 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/913deceb7dfd420b8ea526bc5e0c3aab is 50, key is test_row_0/C:col10/1732371911326/Put/seqid=0 2024-11-23T14:25:11,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741849_1025 (size=12001) 2024-11-23T14:25:11,891 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/e8caa35907ed427fa90bea148eb97606 2024-11-23T14:25:11,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741850_1026 (size=12104) 2024-11-23T14:25:11,922 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/913deceb7dfd420b8ea526bc5e0c3aab as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/913deceb7dfd420b8ea526bc5e0c3aab 2024-11-23T14:25:11,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:11,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34082 deadline: 1732371971920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:11,929 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:11,929 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:11,930 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:11,934 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/8b7949c65c0e4b9789789962f16f884c is 50, key is test_row_0/C:col10/1732371911373/Put/seqid=0 2024-11-23T14:25:11,943 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:11,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1732371971932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:11,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:11,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:11,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:11,948 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5d1382f36edf9c126fa17e1f99c7f3fe/C of 5d1382f36edf9c126fa17e1f99c7f3fe into 913deceb7dfd420b8ea526bc5e0c3aab(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:25:11,948 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d1382f36edf9c126fa17e1f99c7f3fe: 2024-11-23T14:25:11,948 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., storeName=5d1382f36edf9c126fa17e1f99c7f3fe/C, priority=13, startTime=1732371911594; duration=0sec 2024-11-23T14:25:11,949 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:25:11,949 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d1382f36edf9c126fa17e1f99c7f3fe:C 2024-11-23T14:25:11,949 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:11,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34034 deadline: 1732371971941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:11,950 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:11,950 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:11,951 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:11,950 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:11,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34062 deadline: 1732371971941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:11,951 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:11,951 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:11,951 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:11,951 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:11,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34106 deadline: 1732371971942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:11,952 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:11,952 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:11,952 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:11,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741851_1027 (size=12001) 2024-11-23T14:25:11,960 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/8b7949c65c0e4b9789789962f16f884c 2024-11-23T14:25:11,978 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/062b372f06be4727bc2b62598dbbbfcb as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/062b372f06be4727bc2b62598dbbbfcb 2024-11-23T14:25:11,999 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/062b372f06be4727bc2b62598dbbbfcb, entries=200, sequenceid=75, filesize=14.0 K 2024-11-23T14:25:12,004 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/e8caa35907ed427fa90bea148eb97606 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/e8caa35907ed427fa90bea148eb97606 2024-11-23T14:25:12,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-23T14:25:12,021 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T14:25:12,022 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-23T14:25:12,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-23T14:25:12,025 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/e8caa35907ed427fa90bea148eb97606, entries=150, sequenceid=75, filesize=11.7 K 2024-11-23T14:25:12,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-23T14:25:12,027 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T14:25:12,028 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/8b7949c65c0e4b9789789962f16f884c as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/8b7949c65c0e4b9789789962f16f884c 2024-11-23T14:25:12,029 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T14:25:12,029 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T14:25:12,049 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/8b7949c65c0e4b9789789962f16f884c, entries=150, sequenceid=75, filesize=11.7 K 2024-11-23T14:25:12,051 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 5d1382f36edf9c126fa17e1f99c7f3fe in 285ms, sequenceid=75, compaction requested=false 2024-11-23T14:25:12,051 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5d1382f36edf9c126fa17e1f99c7f3fe: 2024-11-23T14:25:12,129 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/e2ed9146f9c644d197142d726542a25e as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/e2ed9146f9c644d197142d726542a25e 2024-11-23T14:25:12,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-23T14:25:12,144 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T14:25:12,145 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5d1382f36edf9c126fa17e1f99c7f3fe 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-23T14:25:12,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on 5d1382f36edf9c126fa17e1f99c7f3fe 2024-11-23T14:25:12,152 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5d1382f36edf9c126fa17e1f99c7f3fe/B of 5d1382f36edf9c126fa17e1f99c7f3fe into e2ed9146f9c644d197142d726542a25e(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:25:12,153 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d1382f36edf9c126fa17e1f99c7f3fe: 2024-11-23T14:25:12,153 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., storeName=5d1382f36edf9c126fa17e1f99c7f3fe/B, priority=13, startTime=1732371911593; duration=0sec 2024-11-23T14:25:12,153 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:25:12,153 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d1382f36edf9c126fa17e1f99c7f3fe:B 2024-11-23T14:25:12,162 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/5fdb8e01052c41a8a692ff3ee5ffebda is 50, key is test_row_0/A:col10/1732371911790/Put/seqid=0 2024-11-23T14:25:12,184 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-23T14:25:12,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:12,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. as already flushing 2024-11-23T14:25:12,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:12,185 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:12,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:12,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741852_1028 (size=14341) 2024-11-23T14:25:12,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:12,188 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/5fdb8e01052c41a8a692ff3ee5ffebda 2024-11-23T14:25:12,219 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/dea0eabfc5094034865beb8d2ebc7f68 is 50, key is test_row_0/B:col10/1732371911790/Put/seqid=0 2024-11-23T14:25:12,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741853_1029 (size=12001) 2024-11-23T14:25:12,294 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:12,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34106 deadline: 1732371972286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:12,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:12,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:12,295 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:12,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:12,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1732371972287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:12,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:12,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:12,296 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:12,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:12,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34062 deadline: 1732371972287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:12,297 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:12,297 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:12,297 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:12,297 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:12,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34034 deadline: 1732371972289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:12,298 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:12,298 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:12,298 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:12,298 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:12,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34082 deadline: 1732371972289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:12,299 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:12,299 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:12,299 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:12,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-23T14:25:12,341 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-23T14:25:12,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:12,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. as already flushing 2024-11-23T14:25:12,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:12,342 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:12,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:12,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:12,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:12,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34062 deadline: 1732371972402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:12,404 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:12,404 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:12,404 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:12,404 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:12,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34106 deadline: 1732371972402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:12,405 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:12,405 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:12,405 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:12,405 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:12,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1732371972403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:12,406 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:12,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34034 deadline: 1732371972403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:12,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:12,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:12,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:12,409 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:12,409 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:12,409 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:12,412 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:12,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34082 deadline: 1732371972410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:12,413 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:12,413 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:12,413 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:12,502 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-23T14:25:12,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:12,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. as already flushing 2024-11-23T14:25:12,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:12,502 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:12,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:12,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:12,550 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-23T14:25:12,615 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:12,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34062 deadline: 1732371972611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:12,616 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:12,616 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:12,617 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:12,617 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:12,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34106 deadline: 1732371972611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:12,619 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:12,619 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:12,619 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:12,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:12,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1732371972621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:12,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:12,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34034 deadline: 1732371972621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:12,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:12,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34082 deadline: 1732371972621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:12,630 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:12,630 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:12,630 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:12,630 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:12,630 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:12,631 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:12,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:12,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:12,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:12,639 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-23T14:25:12,639 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-23T14:25:12,640 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/dea0eabfc5094034865beb8d2ebc7f68 2024-11-23T14:25:12,642 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-23T14:25:12,642 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-23T14:25:12,643 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-23T14:25:12,643 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-23T14:25:12,644 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-23T14:25:12,644 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-23T14:25:12,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-23T14:25:12,663 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/909fb2e98d934d6aa4cfec4332bbf270 is 50, key is test_row_0/C:col10/1732371911790/Put/seqid=0 2024-11-23T14:25:12,669 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-23T14:25:12,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:12,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. as already flushing 2024-11-23T14:25:12,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:12,670 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:12,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:12,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:12,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741854_1030 (size=12001) 2024-11-23T14:25:12,702 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/909fb2e98d934d6aa4cfec4332bbf270 2024-11-23T14:25:12,719 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/5fdb8e01052c41a8a692ff3ee5ffebda as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/5fdb8e01052c41a8a692ff3ee5ffebda 2024-11-23T14:25:12,733 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/5fdb8e01052c41a8a692ff3ee5ffebda, entries=200, sequenceid=88, filesize=14.0 K 2024-11-23T14:25:12,744 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/dea0eabfc5094034865beb8d2ebc7f68 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/dea0eabfc5094034865beb8d2ebc7f68 2024-11-23T14:25:12,763 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/dea0eabfc5094034865beb8d2ebc7f68, entries=150, sequenceid=88, filesize=11.7 K 2024-11-23T14:25:12,766 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/909fb2e98d934d6aa4cfec4332bbf270 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/909fb2e98d934d6aa4cfec4332bbf270 2024-11-23T14:25:12,779 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/909fb2e98d934d6aa4cfec4332bbf270, entries=150, sequenceid=88, filesize=11.7 K 2024-11-23T14:25:12,782 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 5d1382f36edf9c126fa17e1f99c7f3fe in 636ms, sequenceid=88, compaction requested=true 2024-11-23T14:25:12,782 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5d1382f36edf9c126fa17e1f99c7f3fe: 2024-11-23T14:25:12,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d1382f36edf9c126fa17e1f99c7f3fe:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T14:25:12,782 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:25:12,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:25:12,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d1382f36edf9c126fa17e1f99c7f3fe:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T14:25:12,782 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:25:12,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:25:12,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d1382f36edf9c126fa17e1f99c7f3fe:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T14:25:12,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:25:12,786 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:25:12,786 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1541): 5d1382f36edf9c126fa17e1f99c7f3fe/B is initiating minor compaction (all files) 2024-11-23T14:25:12,786 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d1382f36edf9c126fa17e1f99c7f3fe/B in TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:12,786 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/e2ed9146f9c644d197142d726542a25e, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/e8caa35907ed427fa90bea148eb97606, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/dea0eabfc5094034865beb8d2ebc7f68] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp, totalSize=35.3 K 2024-11-23T14:25:12,787 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40786 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:25:12,787 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1541): 5d1382f36edf9c126fa17e1f99c7f3fe/A is initiating minor compaction (all files) 2024-11-23T14:25:12,787 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d1382f36edf9c126fa17e1f99c7f3fe/A in TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:12,787 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/130a7944e2004a1a8cc64d56b2a61d9f, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/062b372f06be4727bc2b62598dbbbfcb, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/5fdb8e01052c41a8a692ff3ee5ffebda] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp, totalSize=39.8 K 2024-11-23T14:25:12,788 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting e2ed9146f9c644d197142d726542a25e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732371910098 2024-11-23T14:25:12,789 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 130a7944e2004a1a8cc64d56b2a61d9f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732371910098 2024-11-23T14:25:12,789 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting e8caa35907ed427fa90bea148eb97606, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1732371911373 2024-11-23T14:25:12,790 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 062b372f06be4727bc2b62598dbbbfcb, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1732371911373 2024-11-23T14:25:12,790 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting dea0eabfc5094034865beb8d2ebc7f68, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1732371911790 2024-11-23T14:25:12,790 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5fdb8e01052c41a8a692ff3ee5ffebda, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1732371911790 2024-11-23T14:25:12,811 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d1382f36edf9c126fa17e1f99c7f3fe#B#compaction#18 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:25:12,812 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/6098ef6602ad46bf962d5d4be1d40884 is 50, key is test_row_0/B:col10/1732371911790/Put/seqid=0 2024-11-23T14:25:12,827 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-23T14:25:12,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:12,828 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 5d1382f36edf9c126fa17e1f99c7f3fe 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-23T14:25:12,832 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d1382f36edf9c126fa17e1f99c7f3fe#A#compaction#19 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:25:12,833 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/7a0fcdf1079246b3aab768384e61dba6 is 50, key is test_row_0/A:col10/1732371911790/Put/seqid=0 2024-11-23T14:25:12,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741855_1031 (size=12207) 2024-11-23T14:25:12,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/9f6fe0ce224941f083175f80ff7d67ed is 50, key is test_row_0/A:col10/1732371912214/Put/seqid=0 2024-11-23T14:25:12,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741856_1032 (size=12207) 2024-11-23T14:25:12,876 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/6098ef6602ad46bf962d5d4be1d40884 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/6098ef6602ad46bf962d5d4be1d40884 2024-11-23T14:25:12,888 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/7a0fcdf1079246b3aab768384e61dba6 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/7a0fcdf1079246b3aab768384e61dba6 2024-11-23T14:25:12,906 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5d1382f36edf9c126fa17e1f99c7f3fe/B of 5d1382f36edf9c126fa17e1f99c7f3fe into 6098ef6602ad46bf962d5d4be1d40884(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:25:12,907 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d1382f36edf9c126fa17e1f99c7f3fe: 2024-11-23T14:25:12,907 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., storeName=5d1382f36edf9c126fa17e1f99c7f3fe/B, priority=13, startTime=1732371912782; duration=0sec 2024-11-23T14:25:12,907 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5d1382f36edf9c126fa17e1f99c7f3fe/A of 5d1382f36edf9c126fa17e1f99c7f3fe into 7a0fcdf1079246b3aab768384e61dba6(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:25:12,907 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d1382f36edf9c126fa17e1f99c7f3fe: 2024-11-23T14:25:12,907 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:25:12,907 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., storeName=5d1382f36edf9c126fa17e1f99c7f3fe/A, priority=13, startTime=1732371912782; duration=0sec 2024-11-23T14:25:12,907 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d1382f36edf9c126fa17e1f99c7f3fe:B 2024-11-23T14:25:12,907 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:25:12,908 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:25:12,908 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d1382f36edf9c126fa17e1f99c7f3fe:A 2024-11-23T14:25:12,912 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:25:12,912 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1541): 5d1382f36edf9c126fa17e1f99c7f3fe/C is initiating minor compaction (all files) 2024-11-23T14:25:12,913 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d1382f36edf9c126fa17e1f99c7f3fe/C in TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:12,913 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/913deceb7dfd420b8ea526bc5e0c3aab, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/8b7949c65c0e4b9789789962f16f884c, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/909fb2e98d934d6aa4cfec4332bbf270] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp, totalSize=35.3 K 2024-11-23T14:25:12,914 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 913deceb7dfd420b8ea526bc5e0c3aab, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732371910098 2024-11-23T14:25:12,915 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b7949c65c0e4b9789789962f16f884c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1732371911373 2024-11-23T14:25:12,919 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 909fb2e98d934d6aa4cfec4332bbf270, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1732371911790 2024-11-23T14:25:12,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741857_1033 (size=12001) 2024-11-23T14:25:12,928 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/9f6fe0ce224941f083175f80ff7d67ed 2024-11-23T14:25:12,935 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. as already flushing 2024-11-23T14:25:12,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on 5d1382f36edf9c126fa17e1f99c7f3fe 2024-11-23T14:25:12,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/65c6b76d14dc4f23867d16e05c184798 is 50, key is test_row_0/B:col10/1732371912214/Put/seqid=0 2024-11-23T14:25:12,955 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:12,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:12,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34062 deadline: 1732371972947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:12,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34106 deadline: 1732371972950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:12,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:12,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1732371972951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:12,959 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d1382f36edf9c126fa17e1f99c7f3fe#C#compaction#22 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:25:12,960 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:12,960 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:12,960 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:12,960 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/4af6fca728f64c45bce2537b306ad802 is 50, key is test_row_0/C:col10/1732371911790/Put/seqid=0 2024-11-23T14:25:12,961 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:12,961 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:12,961 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:12,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:12,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34034 deadline: 1732371972952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:12,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:12,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:12,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:12,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:12,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34082 deadline: 1732371972954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:12,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:12,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:12,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:12,965 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:12,965 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:12,965 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:12,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741858_1034 (size=12001) 2024-11-23T14:25:12,999 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/65c6b76d14dc4f23867d16e05c184798 2024-11-23T14:25:13,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741859_1035 (size=12207) 2024-11-23T14:25:13,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/aa4b01178fd544d38c96b6a4ad6cc4b6 is 50, key is test_row_0/C:col10/1732371912214/Put/seqid=0 2024-11-23T14:25:13,036 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/4af6fca728f64c45bce2537b306ad802 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/4af6fca728f64c45bce2537b306ad802 2024-11-23T14:25:13,049 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5d1382f36edf9c126fa17e1f99c7f3fe/C of 5d1382f36edf9c126fa17e1f99c7f3fe into 4af6fca728f64c45bce2537b306ad802(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:25:13,050 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d1382f36edf9c126fa17e1f99c7f3fe: 2024-11-23T14:25:13,050 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., storeName=5d1382f36edf9c126fa17e1f99c7f3fe/C, priority=13, startTime=1732371912782; duration=0sec 2024-11-23T14:25:13,051 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:25:13,051 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d1382f36edf9c126fa17e1f99c7f3fe:C 2024-11-23T14:25:13,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741860_1036 (size=12001) 2024-11-23T14:25:13,060 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/aa4b01178fd544d38c96b6a4ad6cc4b6 2024-11-23T14:25:13,072 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:13,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1732371973071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:13,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:13,074 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:13,073 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:13,074 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:13,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34062 deadline: 1732371973071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:13,075 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:13,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:13,075 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:13,075 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:13,075 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:13,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34082 deadline: 1732371973072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:13,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34034 deadline: 1732371973072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:13,076 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:13,076 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:13,076 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:13,076 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:13,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/9f6fe0ce224941f083175f80ff7d67ed as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/9f6fe0ce224941f083175f80ff7d67ed 2024-11-23T14:25:13,076 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:13,076 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:13,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:13,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34106 deadline: 1732371973073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:13,077 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:13,077 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:13,077 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:13,092 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/9f6fe0ce224941f083175f80ff7d67ed, entries=150, sequenceid=115, filesize=11.7 K 2024-11-23T14:25:13,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/65c6b76d14dc4f23867d16e05c184798 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/65c6b76d14dc4f23867d16e05c184798 2024-11-23T14:25:13,115 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/65c6b76d14dc4f23867d16e05c184798, entries=150, sequenceid=115, filesize=11.7 K 2024-11-23T14:25:13,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/aa4b01178fd544d38c96b6a4ad6cc4b6 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/aa4b01178fd544d38c96b6a4ad6cc4b6 2024-11-23T14:25:13,133 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/aa4b01178fd544d38c96b6a4ad6cc4b6, entries=150, sequenceid=115, filesize=11.7 K 2024-11-23T14:25:13,135 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 5d1382f36edf9c126fa17e1f99c7f3fe in 306ms, sequenceid=115, compaction requested=false 2024-11-23T14:25:13,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 5d1382f36edf9c126fa17e1f99c7f3fe: 2024-11-23T14:25:13,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:13,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-23T14:25:13,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-23T14:25:13,143 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-23T14:25:13,143 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1090 sec 2024-11-23T14:25:13,149 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.1220 sec 2024-11-23T14:25:13,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-23T14:25:13,161 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T14:25:13,162 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-23T14:25:13,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-23T14:25:13,169 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T14:25:13,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-23T14:25:13,171 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T14:25:13,171 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T14:25:13,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-23T14:25:13,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on 5d1382f36edf9c126fa17e1f99c7f3fe 2024-11-23T14:25:13,284 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T14:25:13,285 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5d1382f36edf9c126fa17e1f99c7f3fe 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-23T14:25:13,296 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/0489d8084889467c922077b874ea49b9 is 50, key is test_row_0/A:col10/1732371913282/Put/seqid=0 2024-11-23T14:25:13,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741861_1037 (size=12051) 2024-11-23T14:25:13,318 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/0489d8084889467c922077b874ea49b9 2024-11-23T14:25:13,329 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-23T14:25:13,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:13,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. as already flushing 2024-11-23T14:25:13,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:13,329 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:13,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:13,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:13,343 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/944e31039d4a444cafbeee6ed03b1653 is 50, key is test_row_0/B:col10/1732371913282/Put/seqid=0 2024-11-23T14:25:13,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:13,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34106 deadline: 1732371973340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:13,357 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:13,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:13,357 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:13,357 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:13,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34062 deadline: 1732371973354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:13,358 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:13,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34034 deadline: 1732371973349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:13,358 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:13,359 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:13,358 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:13,359 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:13,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34082 deadline: 1732371973358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:13,360 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:13,360 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:13,360 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:13,360 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:13,360 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:13,360 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:13,362 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:13,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1732371973358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:13,363 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:13,363 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:13,363 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:13,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741862_1038 (size=12051) 2024-11-23T14:25:13,367 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/944e31039d4a444cafbeee6ed03b1653 2024-11-23T14:25:13,414 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/a54913e8bc234be080d5ecf6c488c686 is 50, key is test_row_0/C:col10/1732371913282/Put/seqid=0 2024-11-23T14:25:13,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741863_1039 (size=12051) 2024-11-23T14:25:13,447 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/a54913e8bc234be080d5ecf6c488c686 2024-11-23T14:25:13,461 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/0489d8084889467c922077b874ea49b9 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/0489d8084889467c922077b874ea49b9 2024-11-23T14:25:13,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:13,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34106 deadline: 1732371973461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:13,463 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:13,463 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:13,463 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:13,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:13,471 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/0489d8084889467c922077b874ea49b9, entries=150, sequenceid=129, filesize=11.8 K 2024-11-23T14:25:13,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34082 deadline: 1732371973470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:13,472 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:13,472 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:13,472 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:13,474 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:13,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34034 deadline: 1732371973471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:13,475 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:13,475 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:13,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1732371973471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:13,475 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:13,476 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:13,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:13,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:13,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:13,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:13,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34062 deadline: 1732371973471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:13,477 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/944e31039d4a444cafbeee6ed03b1653 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/944e31039d4a444cafbeee6ed03b1653 2024-11-23T14:25:13,477 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:13,477 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:13,477 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:13,485 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-23T14:25:13,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:13,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. as already flushing 2024-11-23T14:25:13,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:13,486 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:13,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:13,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:13,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-23T14:25:13,500 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/944e31039d4a444cafbeee6ed03b1653, entries=150, sequenceid=129, filesize=11.8 K 2024-11-23T14:25:13,509 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/a54913e8bc234be080d5ecf6c488c686 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/a54913e8bc234be080d5ecf6c488c686 2024-11-23T14:25:13,529 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/a54913e8bc234be080d5ecf6c488c686, entries=150, sequenceid=129, filesize=11.8 K 2024-11-23T14:25:13,531 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 5d1382f36edf9c126fa17e1f99c7f3fe in 246ms, sequenceid=129, compaction requested=true 2024-11-23T14:25:13,531 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5d1382f36edf9c126fa17e1f99c7f3fe: 2024-11-23T14:25:13,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d1382f36edf9c126fa17e1f99c7f3fe:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T14:25:13,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:25:13,531 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:25:13,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d1382f36edf9c126fa17e1f99c7f3fe:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T14:25:13,531 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:25:13,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:25:13,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d1382f36edf9c126fa17e1f99c7f3fe:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T14:25:13,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:25:13,533 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:25:13,533 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1541): 5d1382f36edf9c126fa17e1f99c7f3fe/B is initiating minor compaction (all files) 2024-11-23T14:25:13,533 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d1382f36edf9c126fa17e1f99c7f3fe/B in TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:13,533 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/6098ef6602ad46bf962d5d4be1d40884, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/65c6b76d14dc4f23867d16e05c184798, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/944e31039d4a444cafbeee6ed03b1653] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp, totalSize=35.4 K 2024-11-23T14:25:13,534 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:25:13,534 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1541): 5d1382f36edf9c126fa17e1f99c7f3fe/A is initiating minor compaction (all files) 2024-11-23T14:25:13,534 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d1382f36edf9c126fa17e1f99c7f3fe/A in TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:13,534 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 6098ef6602ad46bf962d5d4be1d40884, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1732371911790 2024-11-23T14:25:13,534 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/7a0fcdf1079246b3aab768384e61dba6, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/9f6fe0ce224941f083175f80ff7d67ed, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/0489d8084889467c922077b874ea49b9] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp, totalSize=35.4 K 2024-11-23T14:25:13,535 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 65c6b76d14dc4f23867d16e05c184798, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732371912214 2024-11-23T14:25:13,535 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a0fcdf1079246b3aab768384e61dba6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1732371911790 2024-11-23T14:25:13,536 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 944e31039d4a444cafbeee6ed03b1653, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1732371912944 2024-11-23T14:25:13,536 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f6fe0ce224941f083175f80ff7d67ed, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732371912214 2024-11-23T14:25:13,537 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0489d8084889467c922077b874ea49b9, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1732371912944 2024-11-23T14:25:13,579 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d1382f36edf9c126fa17e1f99c7f3fe#B#compaction#27 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:25:13,580 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/1f528556cc714ccab2ed6496fa356ae9 is 50, key is test_row_0/B:col10/1732371913282/Put/seqid=0 2024-11-23T14:25:13,593 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d1382f36edf9c126fa17e1f99c7f3fe#A#compaction#28 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:25:13,594 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/e34d3a2381944f4b9f56ec81f334cd2c is 50, key is test_row_0/A:col10/1732371913282/Put/seqid=0 2024-11-23T14:25:13,642 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-23T14:25:13,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:13,643 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 5d1382f36edf9c126fa17e1f99c7f3fe 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-23T14:25:13,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741864_1040 (size=12359) 2024-11-23T14:25:13,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741865_1041 (size=12359) 2024-11-23T14:25:13,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on 5d1382f36edf9c126fa17e1f99c7f3fe 2024-11-23T14:25:13,699 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. as already flushing 2024-11-23T14:25:13,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/7e2afbcba7604e4c8e6371854744ad95 is 50, key is test_row_0/A:col10/1732371913351/Put/seqid=0 2024-11-23T14:25:13,723 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:13,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34062 deadline: 1732371973705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:13,727 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:13,727 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:13,727 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:13,728 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:13,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1732371973713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:13,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:13,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:13,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:13,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:13,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34034 deadline: 1732371973717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:13,729 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:13,729 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:13,729 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:13,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:13,730 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:13,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34082 deadline: 1732371973725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:13,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34106 deadline: 1732371973721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:13,731 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:13,731 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:13,731 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:13,731 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:13,731 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:13,731 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:13,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741866_1042 (size=12151) 2024-11-23T14:25:13,739 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/7e2afbcba7604e4c8e6371854744ad95 2024-11-23T14:25:13,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/8d538adcc3c54b99a55ef245f5f7b7de is 50, key is test_row_0/B:col10/1732371913351/Put/seqid=0 2024-11-23T14:25:13,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741867_1043 (size=12151) 2024-11-23T14:25:13,784 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/8d538adcc3c54b99a55ef245f5f7b7de 2024-11-23T14:25:13,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-23T14:25:13,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/c288c7a7b1db45dc9ea529338012354e is 50, key is test_row_0/C:col10/1732371913351/Put/seqid=0 2024-11-23T14:25:13,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741868_1044 (size=12151) 2024-11-23T14:25:13,816 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/c288c7a7b1db45dc9ea529338012354e 2024-11-23T14:25:13,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/7e2afbcba7604e4c8e6371854744ad95 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/7e2afbcba7604e4c8e6371854744ad95 2024-11-23T14:25:13,835 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:13,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34062 deadline: 1732371973830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:13,836 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:13,836 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:13,836 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:13,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1732371973830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:13,836 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:13,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:13,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:13,837 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:13,840 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/7e2afbcba7604e4c8e6371854744ad95, entries=150, sequenceid=155, filesize=11.9 K 2024-11-23T14:25:13,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/8d538adcc3c54b99a55ef245f5f7b7de as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/8d538adcc3c54b99a55ef245f5f7b7de 2024-11-23T14:25:13,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:13,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34034 deadline: 1732371973841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:13,843 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:13,843 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:13,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:13,843 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:13,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34082 deadline: 1732371973842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:13,844 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:13,844 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:13,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:13,844 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:13,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34106 deadline: 1732371973842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:13,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:13,845 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:13,845 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:13,853 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/8d538adcc3c54b99a55ef245f5f7b7de, entries=150, sequenceid=155, filesize=11.9 K 2024-11-23T14:25:13,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/c288c7a7b1db45dc9ea529338012354e as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/c288c7a7b1db45dc9ea529338012354e 2024-11-23T14:25:13,867 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/c288c7a7b1db45dc9ea529338012354e, entries=150, sequenceid=155, filesize=11.9 K 2024-11-23T14:25:13,869 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 5d1382f36edf9c126fa17e1f99c7f3fe in 226ms, sequenceid=155, compaction requested=true 2024-11-23T14:25:13,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 5d1382f36edf9c126fa17e1f99c7f3fe: 2024-11-23T14:25:13,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:13,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-23T14:25:13,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-23T14:25:13,879 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-23T14:25:13,879 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 703 msec 2024-11-23T14:25:13,885 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 718 msec 2024-11-23T14:25:14,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on 5d1382f36edf9c126fa17e1f99c7f3fe 2024-11-23T14:25:14,046 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T14:25:14,046 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5d1382f36edf9c126fa17e1f99c7f3fe 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-23T14:25:14,060 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/1f528556cc714ccab2ed6496fa356ae9 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/1f528556cc714ccab2ed6496fa356ae9 2024-11-23T14:25:14,065 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/84b54d10afc74de69276be79ea1b3d6f is 50, key is test_row_0/A:col10/1732371914042/Put/seqid=0 2024-11-23T14:25:14,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741869_1045 (size=16931) 2024-11-23T14:25:14,080 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5d1382f36edf9c126fa17e1f99c7f3fe/B of 5d1382f36edf9c126fa17e1f99c7f3fe into 1f528556cc714ccab2ed6496fa356ae9(size=12.1 K), total size for store is 23.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:25:14,081 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d1382f36edf9c126fa17e1f99c7f3fe: 2024-11-23T14:25:14,081 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., storeName=5d1382f36edf9c126fa17e1f99c7f3fe/B, priority=13, startTime=1732371913531; duration=0sec 2024-11-23T14:25:14,081 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/84b54d10afc74de69276be79ea1b3d6f 2024-11-23T14:25:14,081 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:25:14,081 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d1382f36edf9c126fa17e1f99c7f3fe:B 2024-11-23T14:25:14,081 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T14:25:14,085 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48410 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T14:25:14,085 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1541): 5d1382f36edf9c126fa17e1f99c7f3fe/C is initiating minor compaction (all files) 2024-11-23T14:25:14,085 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d1382f36edf9c126fa17e1f99c7f3fe/C in TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:14,086 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/4af6fca728f64c45bce2537b306ad802, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/aa4b01178fd544d38c96b6a4ad6cc4b6, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/a54913e8bc234be080d5ecf6c488c686, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/c288c7a7b1db45dc9ea529338012354e] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp, totalSize=47.3 K 2024-11-23T14:25:14,086 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 4af6fca728f64c45bce2537b306ad802, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1732371911790 2024-11-23T14:25:14,088 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting aa4b01178fd544d38c96b6a4ad6cc4b6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732371912214 2024-11-23T14:25:14,089 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting a54913e8bc234be080d5ecf6c488c686, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1732371912944 2024-11-23T14:25:14,090 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting c288c7a7b1db45dc9ea529338012354e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732371913348 2024-11-23T14:25:14,093 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/24d7f28c7ae243b5b1f2e38e2ba5b310 is 50, key is test_row_0/B:col10/1732371914042/Put/seqid=0 2024-11-23T14:25:14,109 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:14,109 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/e34d3a2381944f4b9f56ec81f334cd2c as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/e34d3a2381944f4b9f56ec81f334cd2c 2024-11-23T14:25:14,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34106 deadline: 1732371974095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:14,110 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:14,110 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:14,110 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:14,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:14,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34034 deadline: 1732371974097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:14,111 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:14,111 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:14,111 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:14,111 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:14,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34062 deadline: 1732371974101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:14,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:14,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34082 deadline: 1732371974101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:14,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:14,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:14,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:14,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1732371974103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:14,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:14,113 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:14,113 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:14,113 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:14,113 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:14,113 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:14,113 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:14,124 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5d1382f36edf9c126fa17e1f99c7f3fe/A of 5d1382f36edf9c126fa17e1f99c7f3fe into e34d3a2381944f4b9f56ec81f334cd2c(size=12.1 K), total size for store is 23.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:25:14,124 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d1382f36edf9c126fa17e1f99c7f3fe: 2024-11-23T14:25:14,124 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., storeName=5d1382f36edf9c126fa17e1f99c7f3fe/A, priority=13, startTime=1732371913531; duration=0sec 2024-11-23T14:25:14,124 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:25:14,124 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d1382f36edf9c126fa17e1f99c7f3fe:A 2024-11-23T14:25:14,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741870_1046 (size=12151) 2024-11-23T14:25:14,148 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/24d7f28c7ae243b5b1f2e38e2ba5b310 2024-11-23T14:25:14,153 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d1382f36edf9c126fa17e1f99c7f3fe#C#compaction#34 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:25:14,154 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/291dc2ca16aa4a8b85e25bbdd0b44025 is 50, key is test_row_0/C:col10/1732371913351/Put/seqid=0 2024-11-23T14:25:14,164 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/3ef064e14c8b42a4a677b25d545cc65a is 50, key is test_row_0/C:col10/1732371914042/Put/seqid=0 2024-11-23T14:25:14,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741872_1048 (size=12493) 2024-11-23T14:25:14,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741871_1047 (size=12151) 2024-11-23T14:25:14,212 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/3ef064e14c8b42a4a677b25d545cc65a 2024-11-23T14:25:14,212 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/291dc2ca16aa4a8b85e25bbdd0b44025 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/291dc2ca16aa4a8b85e25bbdd0b44025 2024-11-23T14:25:14,221 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:14,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34106 deadline: 1732371974220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:14,222 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:14,222 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:14,222 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:14,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:14,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34062 deadline: 1732371974220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:14,226 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:14,226 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:14,226 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:14,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:14,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34082 deadline: 1732371974221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:14,227 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:14,227 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:14,227 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:14,227 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:14,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34034 deadline: 1732371974221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:14,228 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:14,228 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:14,228 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:14,228 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/84b54d10afc74de69276be79ea1b3d6f as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/84b54d10afc74de69276be79ea1b3d6f 2024-11-23T14:25:14,229 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:14,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1732371974221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:14,230 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:14,230 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:14,230 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:14,235 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5d1382f36edf9c126fa17e1f99c7f3fe/C of 5d1382f36edf9c126fa17e1f99c7f3fe into 291dc2ca16aa4a8b85e25bbdd0b44025(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:25:14,235 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d1382f36edf9c126fa17e1f99c7f3fe: 2024-11-23T14:25:14,235 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., storeName=5d1382f36edf9c126fa17e1f99c7f3fe/C, priority=12, startTime=1732371913531; duration=0sec 2024-11-23T14:25:14,235 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:25:14,235 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d1382f36edf9c126fa17e1f99c7f3fe:C 2024-11-23T14:25:14,246 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/84b54d10afc74de69276be79ea1b3d6f, entries=250, sequenceid=167, filesize=16.5 K 2024-11-23T14:25:14,249 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/24d7f28c7ae243b5b1f2e38e2ba5b310 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/24d7f28c7ae243b5b1f2e38e2ba5b310 2024-11-23T14:25:14,274 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/24d7f28c7ae243b5b1f2e38e2ba5b310, entries=150, sequenceid=167, filesize=11.9 K 2024-11-23T14:25:14,278 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/3ef064e14c8b42a4a677b25d545cc65a as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/3ef064e14c8b42a4a677b25d545cc65a 2024-11-23T14:25:14,299 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/3ef064e14c8b42a4a677b25d545cc65a, entries=150, sequenceid=167, filesize=11.9 K 2024-11-23T14:25:14,303 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 5d1382f36edf9c126fa17e1f99c7f3fe in 256ms, sequenceid=167, compaction requested=true 2024-11-23T14:25:14,305 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5d1382f36edf9c126fa17e1f99c7f3fe: 2024-11-23T14:25:14,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d1382f36edf9c126fa17e1f99c7f3fe:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T14:25:14,306 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:25:14,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:25:14,306 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:25:14,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d1382f36edf9c126fa17e1f99c7f3fe:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T14:25:14,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:25:14,308 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41441 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:25:14,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d1382f36edf9c126fa17e1f99c7f3fe:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T14:25:14,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:25:14,308 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1541): 5d1382f36edf9c126fa17e1f99c7f3fe/A is initiating minor compaction (all files) 2024-11-23T14:25:14,308 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d1382f36edf9c126fa17e1f99c7f3fe/A in TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:14,308 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/e34d3a2381944f4b9f56ec81f334cd2c, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/7e2afbcba7604e4c8e6371854744ad95, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/84b54d10afc74de69276be79ea1b3d6f] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp, totalSize=40.5 K 2024-11-23T14:25:14,309 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36661 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:25:14,309 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting e34d3a2381944f4b9f56ec81f334cd2c, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1732371912944 2024-11-23T14:25:14,309 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1541): 5d1382f36edf9c126fa17e1f99c7f3fe/B is initiating minor compaction (all files) 2024-11-23T14:25:14,309 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d1382f36edf9c126fa17e1f99c7f3fe/B in TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:14,310 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/1f528556cc714ccab2ed6496fa356ae9, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/8d538adcc3c54b99a55ef245f5f7b7de, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/24d7f28c7ae243b5b1f2e38e2ba5b310] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp, totalSize=35.8 K 2024-11-23T14:25:14,310 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e2afbcba7604e4c8e6371854744ad95, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732371913348 2024-11-23T14:25:14,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-23T14:25:14,311 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T14:25:14,311 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f528556cc714ccab2ed6496fa356ae9, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1732371912944 2024-11-23T14:25:14,312 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-23T14:25:14,312 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d538adcc3c54b99a55ef245f5f7b7de, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732371913348 2024-11-23T14:25:14,313 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 84b54d10afc74de69276be79ea1b3d6f, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1732371913697 2024-11-23T14:25:14,313 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 24d7f28c7ae243b5b1f2e38e2ba5b310, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1732371913697 2024-11-23T14:25:14,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-23T14:25:14,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-23T14:25:14,319 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T14:25:14,322 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T14:25:14,322 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T14:25:14,343 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d1382f36edf9c126fa17e1f99c7f3fe#B#compaction#36 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:25:14,344 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/09fcb7ca83284ccebbe9b6da9173fdfd is 50, key is test_row_0/B:col10/1732371914042/Put/seqid=0 2024-11-23T14:25:14,356 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d1382f36edf9c126fa17e1f99c7f3fe#A#compaction#37 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:25:14,357 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/57f7b2cae40f44c5af9b2f3ff8bd9fe4 is 50, key is test_row_0/A:col10/1732371914042/Put/seqid=0 2024-11-23T14:25:14,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741873_1049 (size=12561) 2024-11-23T14:25:14,397 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/09fcb7ca83284ccebbe9b6da9173fdfd as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/09fcb7ca83284ccebbe9b6da9173fdfd 2024-11-23T14:25:14,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741874_1050 (size=12561) 2024-11-23T14:25:14,419 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5d1382f36edf9c126fa17e1f99c7f3fe/B of 5d1382f36edf9c126fa17e1f99c7f3fe into 09fcb7ca83284ccebbe9b6da9173fdfd(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:25:14,419 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d1382f36edf9c126fa17e1f99c7f3fe: 2024-11-23T14:25:14,419 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., storeName=5d1382f36edf9c126fa17e1f99c7f3fe/B, priority=13, startTime=1732371914306; duration=0sec 2024-11-23T14:25:14,419 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:25:14,419 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d1382f36edf9c126fa17e1f99c7f3fe:B 2024-11-23T14:25:14,419 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-23T14:25:14,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-23T14:25:14,424 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-23T14:25:14,424 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-23T14:25:14,424 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. because compaction request was cancelled 2024-11-23T14:25:14,424 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d1382f36edf9c126fa17e1f99c7f3fe:C 2024-11-23T14:25:14,429 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/57f7b2cae40f44c5af9b2f3ff8bd9fe4 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/57f7b2cae40f44c5af9b2f3ff8bd9fe4 2024-11-23T14:25:14,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on 5d1382f36edf9c126fa17e1f99c7f3fe 2024-11-23T14:25:14,440 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T14:25:14,440 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5d1382f36edf9c126fa17e1f99c7f3fe 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-23T14:25:14,445 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5d1382f36edf9c126fa17e1f99c7f3fe/A of 5d1382f36edf9c126fa17e1f99c7f3fe into 57f7b2cae40f44c5af9b2f3ff8bd9fe4(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:25:14,445 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d1382f36edf9c126fa17e1f99c7f3fe: 2024-11-23T14:25:14,446 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., storeName=5d1382f36edf9c126fa17e1f99c7f3fe/A, priority=13, startTime=1732371914306; duration=0sec 2024-11-23T14:25:14,446 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:25:14,446 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d1382f36edf9c126fa17e1f99c7f3fe:A 2024-11-23T14:25:14,451 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/87877e94345e4534a978a8c207bff523 is 50, key is test_row_0/A:col10/1732371914098/Put/seqid=0 2024-11-23T14:25:14,461 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:14,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1732371974453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:14,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:14,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:14,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:14,465 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:14,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34062 deadline: 1732371974458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:14,466 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:14,466 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:14,466 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:14,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:14,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34034 deadline: 1732371974460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:14,467 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:14,467 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:14,467 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:14,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:14,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34106 deadline: 1732371974461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:14,468 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:14,468 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:14,468 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:14,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:14,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34082 deadline: 1732371974461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:14,469 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:14,469 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:14,469 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:14,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741875_1051 (size=16931) 2024-11-23T14:25:14,478 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/87877e94345e4534a978a8c207bff523 2024-11-23T14:25:14,479 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-23T14:25:14,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:14,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. as already flushing 2024-11-23T14:25:14,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:14,480 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:14,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:14,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:14,495 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/16ef9a7f6de64f5ead7d2738482a1ae7 is 50, key is test_row_0/B:col10/1732371914098/Put/seqid=0 2024-11-23T14:25:14,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741876_1052 (size=12151) 2024-11-23T14:25:14,503 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/16ef9a7f6de64f5ead7d2738482a1ae7 2024-11-23T14:25:14,516 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/82538cafb29f4d2389e1b492fc913de4 is 50, key is test_row_0/C:col10/1732371914098/Put/seqid=0 2024-11-23T14:25:14,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741877_1053 (size=12151) 2024-11-23T14:25:14,523 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/82538cafb29f4d2389e1b492fc913de4 2024-11-23T14:25:14,535 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/87877e94345e4534a978a8c207bff523 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/87877e94345e4534a978a8c207bff523 2024-11-23T14:25:14,545 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/87877e94345e4534a978a8c207bff523, entries=250, sequenceid=197, filesize=16.5 K 2024-11-23T14:25:14,547 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/16ef9a7f6de64f5ead7d2738482a1ae7 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/16ef9a7f6de64f5ead7d2738482a1ae7 2024-11-23T14:25:14,557 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/16ef9a7f6de64f5ead7d2738482a1ae7, entries=150, sequenceid=197, filesize=11.9 K 2024-11-23T14:25:14,559 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/82538cafb29f4d2389e1b492fc913de4 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/82538cafb29f4d2389e1b492fc913de4 2024-11-23T14:25:14,569 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/82538cafb29f4d2389e1b492fc913de4, entries=150, sequenceid=197, filesize=11.9 K 2024-11-23T14:25:14,571 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 5d1382f36edf9c126fa17e1f99c7f3fe in 131ms, sequenceid=197, compaction requested=true 2024-11-23T14:25:14,571 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5d1382f36edf9c126fa17e1f99c7f3fe: 2024-11-23T14:25:14,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d1382f36edf9c126fa17e1f99c7f3fe:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T14:25:14,572 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-23T14:25:14,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:25:14,572 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-23T14:25:14,573 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-23T14:25:14,573 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-23T14:25:14,573 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. because compaction request was cancelled 2024-11-23T14:25:14,573 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d1382f36edf9c126fa17e1f99c7f3fe:A 2024-11-23T14:25:14,573 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-23T14:25:14,573 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-23T14:25:14,573 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. because compaction request was cancelled 2024-11-23T14:25:14,574 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d1382f36edf9c126fa17e1f99c7f3fe:B 2024-11-23T14:25:14,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d1382f36edf9c126fa17e1f99c7f3fe:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T14:25:14,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:25:14,574 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:25:14,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d1382f36edf9c126fa17e1f99c7f3fe:C, priority=-2147483648, current under compaction store size is 1 2024-11-23T14:25:14,575 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:25:14,575 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1541): 5d1382f36edf9c126fa17e1f99c7f3fe/C is initiating minor compaction (all files) 2024-11-23T14:25:14,575 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d1382f36edf9c126fa17e1f99c7f3fe/C in TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:14,576 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/291dc2ca16aa4a8b85e25bbdd0b44025, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/3ef064e14c8b42a4a677b25d545cc65a, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/82538cafb29f4d2389e1b492fc913de4] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp, totalSize=35.9 K 2024-11-23T14:25:14,577 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 291dc2ca16aa4a8b85e25bbdd0b44025, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732371913348 2024-11-23T14:25:14,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:25:14,578 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3ef064e14c8b42a4a677b25d545cc65a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1732371913697 2024-11-23T14:25:14,578 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 82538cafb29f4d2389e1b492fc913de4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1732371914097 2024-11-23T14:25:14,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on 5d1382f36edf9c126fa17e1f99c7f3fe 2024-11-23T14:25:14,580 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T14:25:14,581 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5d1382f36edf9c126fa17e1f99c7f3fe 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-23T14:25:14,595 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/03a0f119ad79421d9c2c5b1448df2211 is 50, key is test_row_0/A:col10/1732371914580/Put/seqid=0 2024-11-23T14:25:14,598 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d1382f36edf9c126fa17e1f99c7f3fe#C#compaction#42 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:25:14,599 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/708d9252368d425181330c0696410e23 is 50, key is test_row_0/C:col10/1732371914098/Put/seqid=0 2024-11-23T14:25:14,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-23T14:25:14,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741879_1055 (size=12595) 2024-11-23T14:25:14,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741878_1054 (size=12151) 2024-11-23T14:25:14,635 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-23T14:25:14,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:14,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. as already flushing 2024-11-23T14:25:14,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:14,636 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:14,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:14,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:14,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:14,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34034 deadline: 1732371974626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:14,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:14,640 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:14,640 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:14,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34082 deadline: 1732371974630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:14,640 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:14,640 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:14,640 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:14,641 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:14,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:14,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34106 deadline: 1732371974632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:14,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:14,644 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:14,644 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:14,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1732371974639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:14,644 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:14,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:14,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:14,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:14,645 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:14,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34062 deadline: 1732371974639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:14,647 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:14,647 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:14,647 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:14,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:14,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34034 deadline: 1732371974751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:14,754 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:14,754 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:14,754 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:14,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:14,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34082 deadline: 1732371974752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:14,755 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:14,755 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:14,755 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:14,759 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:14,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34106 deadline: 1732371974756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:14,760 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:14,760 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:14,760 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:14,761 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:14,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1732371974757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:14,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:14,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:14,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:14,763 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:14,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34062 deadline: 1732371974757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:14,763 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:14,764 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:14,764 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:14,791 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-23T14:25:14,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:14,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. as already flushing 2024-11-23T14:25:14,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:14,792 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:14,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:14,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:14,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-23T14:25:14,946 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-23T14:25:14,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:14,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. as already flushing 2024-11-23T14:25:14,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:14,946 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:14,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:14,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:14,965 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:14,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34034 deadline: 1732371974960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:14,966 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:14,966 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:14,966 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:14,966 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:14,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34082 deadline: 1732371974960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:14,967 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:14,967 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:14,967 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:14,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:14,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34106 deadline: 1732371974971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:14,974 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:14,974 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:14,974 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:14,974 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:14,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1732371974971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:14,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:14,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:14,975 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:14,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:14,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34062 deadline: 1732371974971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:14,975 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:14,975 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:14,975 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:15,034 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/03a0f119ad79421d9c2c5b1448df2211 2024-11-23T14:25:15,041 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/708d9252368d425181330c0696410e23 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/708d9252368d425181330c0696410e23 2024-11-23T14:25:15,051 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5d1382f36edf9c126fa17e1f99c7f3fe/C of 5d1382f36edf9c126fa17e1f99c7f3fe into 708d9252368d425181330c0696410e23(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:25:15,051 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d1382f36edf9c126fa17e1f99c7f3fe: 2024-11-23T14:25:15,051 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., storeName=5d1382f36edf9c126fa17e1f99c7f3fe/C, priority=13, startTime=1732371914574; duration=0sec 2024-11-23T14:25:15,051 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:25:15,051 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d1382f36edf9c126fa17e1f99c7f3fe:C 2024-11-23T14:25:15,058 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/4b12670ef5af427ca84413300855af6c is 50, key is test_row_0/B:col10/1732371914580/Put/seqid=0 2024-11-23T14:25:15,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741880_1056 (size=12151) 2024-11-23T14:25:15,078 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/4b12670ef5af427ca84413300855af6c 2024-11-23T14:25:15,090 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/f142299baa26466bb7c6d7f43e4cccd4 is 50, key is test_row_0/C:col10/1732371914580/Put/seqid=0 2024-11-23T14:25:15,100 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-23T14:25:15,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:15,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. as already flushing 2024-11-23T14:25:15,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:15,101 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:15,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:15,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:15,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741881_1057 (size=12151) 2024-11-23T14:25:15,258 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-23T14:25:15,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:15,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. as already flushing 2024-11-23T14:25:15,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:15,258 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:15,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:15,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:15,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:15,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34082 deadline: 1732371975271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:15,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:15,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34034 deadline: 1732371975271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:15,276 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:15,277 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:15,277 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:15,278 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:15,278 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:15,278 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:15,282 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:15,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34106 deadline: 1732371975281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:15,282 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:15,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1732371975281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:15,282 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:15,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34062 deadline: 1732371975281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:15,283 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:15,283 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:15,283 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:15,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:15,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:15,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:15,290 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:15,290 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:15,290 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:15,414 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-23T14:25:15,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:15,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. as already flushing 2024-11-23T14:25:15,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:15,414 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:15,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:15,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:15,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-23T14:25:15,528 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/f142299baa26466bb7c6d7f43e4cccd4 2024-11-23T14:25:15,536 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/03a0f119ad79421d9c2c5b1448df2211 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/03a0f119ad79421d9c2c5b1448df2211 2024-11-23T14:25:15,544 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/03a0f119ad79421d9c2c5b1448df2211, entries=150, sequenceid=212, filesize=11.9 K 2024-11-23T14:25:15,545 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/4b12670ef5af427ca84413300855af6c as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/4b12670ef5af427ca84413300855af6c 2024-11-23T14:25:15,565 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/4b12670ef5af427ca84413300855af6c, entries=150, sequenceid=212, filesize=11.9 K 2024-11-23T14:25:15,567 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/f142299baa26466bb7c6d7f43e4cccd4 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/f142299baa26466bb7c6d7f43e4cccd4 2024-11-23T14:25:15,570 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-23T14:25:15,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:15,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. as already flushing 2024-11-23T14:25:15,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:15,570 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:15,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:15,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:15,578 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/f142299baa26466bb7c6d7f43e4cccd4, entries=150, sequenceid=212, filesize=11.9 K 2024-11-23T14:25:15,579 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 5d1382f36edf9c126fa17e1f99c7f3fe in 999ms, sequenceid=212, compaction requested=true 2024-11-23T14:25:15,579 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5d1382f36edf9c126fa17e1f99c7f3fe: 2024-11-23T14:25:15,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d1382f36edf9c126fa17e1f99c7f3fe:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T14:25:15,580 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:25:15,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:25:15,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d1382f36edf9c126fa17e1f99c7f3fe:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T14:25:15,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:25:15,580 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:25:15,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d1382f36edf9c126fa17e1f99c7f3fe:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T14:25:15,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:25:15,581 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:25:15,581 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1541): 5d1382f36edf9c126fa17e1f99c7f3fe/B is initiating minor compaction (all files) 2024-11-23T14:25:15,581 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d1382f36edf9c126fa17e1f99c7f3fe/B in TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:15,582 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/09fcb7ca83284ccebbe9b6da9173fdfd, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/16ef9a7f6de64f5ead7d2738482a1ae7, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/4b12670ef5af427ca84413300855af6c] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp, totalSize=36.0 K 2024-11-23T14:25:15,582 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41643 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:25:15,582 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1541): 5d1382f36edf9c126fa17e1f99c7f3fe/A is initiating minor compaction (all files) 2024-11-23T14:25:15,582 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d1382f36edf9c126fa17e1f99c7f3fe/A in TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:15,582 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 09fcb7ca83284ccebbe9b6da9173fdfd, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1732371913697 2024-11-23T14:25:15,582 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/57f7b2cae40f44c5af9b2f3ff8bd9fe4, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/87877e94345e4534a978a8c207bff523, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/03a0f119ad79421d9c2c5b1448df2211] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp, totalSize=40.7 K 2024-11-23T14:25:15,583 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 16ef9a7f6de64f5ead7d2738482a1ae7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1732371914097 2024-11-23T14:25:15,583 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 57f7b2cae40f44c5af9b2f3ff8bd9fe4, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1732371913697 2024-11-23T14:25:15,583 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b12670ef5af427ca84413300855af6c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732371914576 2024-11-23T14:25:15,587 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 87877e94345e4534a978a8c207bff523, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1732371914091 2024-11-23T14:25:15,588 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 03a0f119ad79421d9c2c5b1448df2211, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732371914576 2024-11-23T14:25:15,609 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d1382f36edf9c126fa17e1f99c7f3fe#A#compaction#45 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:25:15,610 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/1705333fb04e4a4787d9994f56ac0c53 is 50, key is test_row_0/A:col10/1732371914580/Put/seqid=0 2024-11-23T14:25:15,611 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d1382f36edf9c126fa17e1f99c7f3fe#B#compaction#46 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:25:15,612 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/a2e096fc89f94798a77fe991d139b779 is 50, key is test_row_0/B:col10/1732371914580/Put/seqid=0 2024-11-23T14:25:15,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741882_1058 (size=12663) 2024-11-23T14:25:15,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741883_1059 (size=12663) 2024-11-23T14:25:15,725 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-23T14:25:15,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:15,725 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 5d1382f36edf9c126fa17e1f99c7f3fe 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-23T14:25:15,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/ff53e25d6541408c89424a2aa6a76fed is 50, key is test_row_0/A:col10/1732371914625/Put/seqid=0 2024-11-23T14:25:15,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741884_1060 (size=12151) 2024-11-23T14:25:15,746 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/ff53e25d6541408c89424a2aa6a76fed 2024-11-23T14:25:15,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/85bb31a16194456fa4ce30d5fc61c1f3 is 50, key is test_row_0/B:col10/1732371914625/Put/seqid=0 2024-11-23T14:25:15,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741885_1061 (size=12151) 2024-11-23T14:25:15,793 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/85bb31a16194456fa4ce30d5fc61c1f3 2024-11-23T14:25:15,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on 5d1382f36edf9c126fa17e1f99c7f3fe 2024-11-23T14:25:15,796 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. as already flushing 2024-11-23T14:25:15,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/6d2d95b572e24f24ba3762810dafcc3b is 50, key is test_row_0/C:col10/1732371914625/Put/seqid=0 2024-11-23T14:25:15,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:15,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1732371975816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:15,826 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:15,826 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:15,826 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:15,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:15,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34034 deadline: 1732371975816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:15,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:15,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34062 deadline: 1732371975818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:15,827 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:15,827 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:15,827 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:15,829 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:15,829 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:15,829 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:15,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:15,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34082 deadline: 1732371975828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:15,830 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:15,830 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:15,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:15,831 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:15,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34106 deadline: 1732371975828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:15,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:15,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:15,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:15,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741886_1062 (size=12151) 2024-11-23T14:25:15,839 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/6d2d95b572e24f24ba3762810dafcc3b 2024-11-23T14:25:15,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/ff53e25d6541408c89424a2aa6a76fed as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/ff53e25d6541408c89424a2aa6a76fed 2024-11-23T14:25:15,853 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/ff53e25d6541408c89424a2aa6a76fed, entries=150, sequenceid=235, filesize=11.9 K 2024-11-23T14:25:15,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/85bb31a16194456fa4ce30d5fc61c1f3 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/85bb31a16194456fa4ce30d5fc61c1f3 2024-11-23T14:25:15,864 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/85bb31a16194456fa4ce30d5fc61c1f3, entries=150, sequenceid=235, filesize=11.9 K 2024-11-23T14:25:15,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/6d2d95b572e24f24ba3762810dafcc3b as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/6d2d95b572e24f24ba3762810dafcc3b 2024-11-23T14:25:15,874 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/6d2d95b572e24f24ba3762810dafcc3b, entries=150, sequenceid=235, filesize=11.9 K 2024-11-23T14:25:15,875 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=80.51 KB/82440 for 5d1382f36edf9c126fa17e1f99c7f3fe in 150ms, sequenceid=235, compaction requested=true 2024-11-23T14:25:15,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 5d1382f36edf9c126fa17e1f99c7f3fe: 2024-11-23T14:25:15,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:15,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-23T14:25:15,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-23T14:25:15,883 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-23T14:25:15,883 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5570 sec 2024-11-23T14:25:15,889 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.5740 sec 2024-11-23T14:25:15,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on 5d1382f36edf9c126fa17e1f99c7f3fe 2024-11-23T14:25:15,934 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T14:25:15,935 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5d1382f36edf9c126fa17e1f99c7f3fe 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-23T14:25:15,944 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/ae318f68ac7b463c9e61df53210a72a5 is 50, key is test_row_0/A:col10/1732371915933/Put/seqid=0 2024-11-23T14:25:15,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741887_1063 (size=12151) 2024-11-23T14:25:15,952 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/ae318f68ac7b463c9e61df53210a72a5 2024-11-23T14:25:15,965 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/c26f740ff82c4769a94e0d2dc68800e9 is 50, key is test_row_0/B:col10/1732371915933/Put/seqid=0 2024-11-23T14:25:15,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:15,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1732371975964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:15,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:15,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34082 deadline: 1732371975966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:15,970 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:15,971 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:15,971 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:15,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:15,971 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:15,971 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:15,971 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:15,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34062 deadline: 1732371975968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:15,971 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:15,971 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:15,971 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:15,977 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:15,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34034 deadline: 1732371975970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:15,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741888_1064 (size=12151) 2024-11-23T14:25:15,978 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:15,978 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:15,978 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:15,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34106 deadline: 1732371975970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:15,978 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:15,978 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:15,978 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:15,978 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:16,052 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/1705333fb04e4a4787d9994f56ac0c53 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/1705333fb04e4a4787d9994f56ac0c53 2024-11-23T14:25:16,062 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5d1382f36edf9c126fa17e1f99c7f3fe/A of 5d1382f36edf9c126fa17e1f99c7f3fe into 1705333fb04e4a4787d9994f56ac0c53(size=12.4 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:25:16,062 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d1382f36edf9c126fa17e1f99c7f3fe: 2024-11-23T14:25:16,062 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., storeName=5d1382f36edf9c126fa17e1f99c7f3fe/A, priority=13, startTime=1732371915579; duration=0sec 2024-11-23T14:25:16,062 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:25:16,062 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d1382f36edf9c126fa17e1f99c7f3fe:A 2024-11-23T14:25:16,062 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:25:16,066 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:25:16,066 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1541): 5d1382f36edf9c126fa17e1f99c7f3fe/C is initiating minor compaction (all files) 2024-11-23T14:25:16,067 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d1382f36edf9c126fa17e1f99c7f3fe/C in TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:16,067 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/708d9252368d425181330c0696410e23, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/f142299baa26466bb7c6d7f43e4cccd4, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/6d2d95b572e24f24ba3762810dafcc3b] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp, totalSize=36.0 K 2024-11-23T14:25:16,067 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 708d9252368d425181330c0696410e23, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1732371914097 2024-11-23T14:25:16,068 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting f142299baa26466bb7c6d7f43e4cccd4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732371914576 2024-11-23T14:25:16,068 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6d2d95b572e24f24ba3762810dafcc3b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732371914625 2024-11-23T14:25:16,079 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/a2e096fc89f94798a77fe991d139b779 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/a2e096fc89f94798a77fe991d139b779 2024-11-23T14:25:16,084 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:16,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1732371976080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:16,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:16,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:16,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:16,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:16,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34062 deadline: 1732371976081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:16,086 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:16,086 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:16,086 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:16,086 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:16,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34034 deadline: 1732371976081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:16,087 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:16,087 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:16,087 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:16,087 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:16,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34106 deadline: 1732371976081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:16,087 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:16,087 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:16,088 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:16,088 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:16,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34082 deadline: 1732371976081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:16,088 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:16,088 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:16,088 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:16,091 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5d1382f36edf9c126fa17e1f99c7f3fe/B of 5d1382f36edf9c126fa17e1f99c7f3fe into a2e096fc89f94798a77fe991d139b779(size=12.4 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:25:16,091 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d1382f36edf9c126fa17e1f99c7f3fe: 2024-11-23T14:25:16,091 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., storeName=5d1382f36edf9c126fa17e1f99c7f3fe/B, priority=13, startTime=1732371915580; duration=0sec 2024-11-23T14:25:16,091 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:25:16,091 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d1382f36edf9c126fa17e1f99c7f3fe:B 2024-11-23T14:25:16,105 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d1382f36edf9c126fa17e1f99c7f3fe#C#compaction#52 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:25:16,106 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/f26f8853a26e452289bd01f69b5d29c3 is 50, key is test_row_0/C:col10/1732371914625/Put/seqid=0 2024-11-23T14:25:16,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741889_1065 (size=12697) 2024-11-23T14:25:16,131 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/f26f8853a26e452289bd01f69b5d29c3 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/f26f8853a26e452289bd01f69b5d29c3 2024-11-23T14:25:16,143 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5d1382f36edf9c126fa17e1f99c7f3fe/C of 5d1382f36edf9c126fa17e1f99c7f3fe into f26f8853a26e452289bd01f69b5d29c3(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:25:16,143 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d1382f36edf9c126fa17e1f99c7f3fe: 2024-11-23T14:25:16,143 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., storeName=5d1382f36edf9c126fa17e1f99c7f3fe/C, priority=13, startTime=1732371915580; duration=0sec 2024-11-23T14:25:16,143 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:25:16,143 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d1382f36edf9c126fa17e1f99c7f3fe:C 2024-11-23T14:25:16,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:16,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1732371976290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:16,293 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:16,293 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:16,293 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:16,294 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:16,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34062 deadline: 1732371976290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:16,294 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:16,294 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:16,294 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:16,294 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:16,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34034 deadline: 1732371976290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:16,295 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:16,295 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:16,295 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:16,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:16,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34106 deadline: 1732371976290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:16,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:16,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:16,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:16,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:16,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34082 deadline: 1732371976300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:16,306 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:16,306 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:16,306 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:16,379 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/c26f740ff82c4769a94e0d2dc68800e9 2024-11-23T14:25:16,395 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/d5656e6d9746440497424aedd94656d5 is 50, key is test_row_0/C:col10/1732371915933/Put/seqid=0 2024-11-23T14:25:16,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741890_1066 (size=12151) 2024-11-23T14:25:16,433 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/d5656e6d9746440497424aedd94656d5 2024-11-23T14:25:16,445 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/ae318f68ac7b463c9e61df53210a72a5 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/ae318f68ac7b463c9e61df53210a72a5 2024-11-23T14:25:16,454 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/ae318f68ac7b463c9e61df53210a72a5, entries=150, sequenceid=252, filesize=11.9 K 2024-11-23T14:25:16,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-23T14:25:16,461 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T14:25:16,461 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/c26f740ff82c4769a94e0d2dc68800e9 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/c26f740ff82c4769a94e0d2dc68800e9 2024-11-23T14:25:16,461 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-23T14:25:16,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-23T14:25:16,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-23T14:25:16,470 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T14:25:16,471 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T14:25:16,472 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T14:25:16,478 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/c26f740ff82c4769a94e0d2dc68800e9, entries=150, sequenceid=252, filesize=11.9 K 2024-11-23T14:25:16,479 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/d5656e6d9746440497424aedd94656d5 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/d5656e6d9746440497424aedd94656d5 2024-11-23T14:25:16,488 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/d5656e6d9746440497424aedd94656d5, entries=150, sequenceid=252, filesize=11.9 K 2024-11-23T14:25:16,492 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=114.05 KB/116790 for 5d1382f36edf9c126fa17e1f99c7f3fe in 558ms, sequenceid=252, compaction requested=true 2024-11-23T14:25:16,492 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5d1382f36edf9c126fa17e1f99c7f3fe: 2024-11-23T14:25:16,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d1382f36edf9c126fa17e1f99c7f3fe:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T14:25:16,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:25:16,492 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:25:16,492 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:25:16,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d1382f36edf9c126fa17e1f99c7f3fe:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T14:25:16,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:25:16,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d1382f36edf9c126fa17e1f99c7f3fe:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T14:25:16,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:25:16,494 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:25:16,494 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1541): 5d1382f36edf9c126fa17e1f99c7f3fe/A is initiating minor compaction (all files) 2024-11-23T14:25:16,494 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:25:16,494 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d1382f36edf9c126fa17e1f99c7f3fe/A in TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:16,494 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1541): 5d1382f36edf9c126fa17e1f99c7f3fe/B is initiating minor compaction (all files) 2024-11-23T14:25:16,494 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d1382f36edf9c126fa17e1f99c7f3fe/B in TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:16,494 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/1705333fb04e4a4787d9994f56ac0c53, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/ff53e25d6541408c89424a2aa6a76fed, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/ae318f68ac7b463c9e61df53210a72a5] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp, totalSize=36.1 K 2024-11-23T14:25:16,494 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/a2e096fc89f94798a77fe991d139b779, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/85bb31a16194456fa4ce30d5fc61c1f3, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/c26f740ff82c4769a94e0d2dc68800e9] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp, totalSize=36.1 K 2024-11-23T14:25:16,495 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1705333fb04e4a4787d9994f56ac0c53, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732371914576 2024-11-23T14:25:16,495 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting a2e096fc89f94798a77fe991d139b779, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732371914576 2024-11-23T14:25:16,496 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting ff53e25d6541408c89424a2aa6a76fed, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732371914625 2024-11-23T14:25:16,496 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 85bb31a16194456fa4ce30d5fc61c1f3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732371914625 2024-11-23T14:25:16,496 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae318f68ac7b463c9e61df53210a72a5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732371915816 2024-11-23T14:25:16,497 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting c26f740ff82c4769a94e0d2dc68800e9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732371915816 2024-11-23T14:25:16,524 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d1382f36edf9c126fa17e1f99c7f3fe#B#compaction#54 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:25:16,525 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/ea84ec993cf14f06a2b46aefdf59d440 is 50, key is test_row_0/B:col10/1732371915933/Put/seqid=0 2024-11-23T14:25:16,546 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d1382f36edf9c126fa17e1f99c7f3fe#A#compaction#55 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:25:16,547 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/34163857dc9449ceace9d451fb26d035 is 50, key is test_row_0/A:col10/1732371915933/Put/seqid=0 2024-11-23T14:25:16,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-23T14:25:16,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741892_1068 (size=12765) 2024-11-23T14:25:16,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on 5d1382f36edf9c126fa17e1f99c7f3fe 2024-11-23T14:25:16,607 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T14:25:16,607 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5d1382f36edf9c126fa17e1f99c7f3fe 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-23T14:25:16,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741891_1067 (size=12765) 2024-11-23T14:25:16,615 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/34163857dc9449ceace9d451fb26d035 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/34163857dc9449ceace9d451fb26d035 2024-11-23T14:25:16,625 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/1e9fdc3213164ee9a757f27953fd6c3e is 50, key is test_row_0/A:col10/1732371915968/Put/seqid=0 2024-11-23T14:25:16,630 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/ea84ec993cf14f06a2b46aefdf59d440 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/ea84ec993cf14f06a2b46aefdf59d440 2024-11-23T14:25:16,630 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-23T14:25:16,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:16,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. as already flushing 2024-11-23T14:25:16,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:16,631 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:16,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:16,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:16,641 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5d1382f36edf9c126fa17e1f99c7f3fe/A of 5d1382f36edf9c126fa17e1f99c7f3fe into 34163857dc9449ceace9d451fb26d035(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:25:16,641 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d1382f36edf9c126fa17e1f99c7f3fe: 2024-11-23T14:25:16,641 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., storeName=5d1382f36edf9c126fa17e1f99c7f3fe/A, priority=13, startTime=1732371916492; duration=0sec 2024-11-23T14:25:16,641 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:25:16,641 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d1382f36edf9c126fa17e1f99c7f3fe:A 2024-11-23T14:25:16,641 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-23T14:25:16,644 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5d1382f36edf9c126fa17e1f99c7f3fe/B of 5d1382f36edf9c126fa17e1f99c7f3fe into ea84ec993cf14f06a2b46aefdf59d440(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:25:16,644 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-23T14:25:16,644 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d1382f36edf9c126fa17e1f99c7f3fe: 2024-11-23T14:25:16,644 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-23T14:25:16,644 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., storeName=5d1382f36edf9c126fa17e1f99c7f3fe/B, priority=13, startTime=1732371916492; duration=0sec 2024-11-23T14:25:16,644 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. because compaction request was cancelled 2024-11-23T14:25:16,644 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d1382f36edf9c126fa17e1f99c7f3fe:C 2024-11-23T14:25:16,645 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:25:16,645 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d1382f36edf9c126fa17e1f99c7f3fe:B 2024-11-23T14:25:16,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:16,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:16,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34062 deadline: 1732371976640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:16,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34034 deadline: 1732371976642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:16,652 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:16,652 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:16,653 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:16,653 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:16,653 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:16,653 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:16,653 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:16,653 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:16,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34082 deadline: 1732371976644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:16,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1732371976643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:16,653 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:16,653 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:16,653 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:16,654 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:16,654 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:16,654 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:16,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:16,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34106 deadline: 1732371976652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:16,656 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:16,656 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:16,656 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:16,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741893_1069 (size=12301) 2024-11-23T14:25:16,761 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:16,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34082 deadline: 1732371976760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:16,762 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:16,762 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:16,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:16,762 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:16,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34034 deadline: 1732371976761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:16,763 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:16,763 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:16,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1732371976761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:16,763 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:16,763 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:16,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:16,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:16,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:16,764 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:16,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34062 deadline: 1732371976761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:16,764 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:16,764 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:16,764 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:16,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:16,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34106 deadline: 1732371976761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:16,766 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:16,766 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:16,766 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:16,786 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-23T14:25:16,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:16,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. as already flushing 2024-11-23T14:25:16,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:16,786 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:16,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:16,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:16,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-23T14:25:16,940 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-23T14:25:16,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:16,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. as already flushing 2024-11-23T14:25:16,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:16,941 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:16,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:16,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:16,975 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:16,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34082 deadline: 1732371976970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:16,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:16,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34034 deadline: 1732371976970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:16,976 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:16,976 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:16,976 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:16,977 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:16,977 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:16,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34106 deadline: 1732371976971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:16,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1732371976970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:16,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:16,978 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:16,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:16,978 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:16,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:16,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:16,978 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:16,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34062 deadline: 1732371976971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:16,978 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:16,978 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:16,978 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:16,978 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:16,978 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:16,978 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:17,068 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/1e9fdc3213164ee9a757f27953fd6c3e 2024-11-23T14:25:17,094 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-23T14:25:17,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:17,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. as already flushing 2024-11-23T14:25:17,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:17,095 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:17,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:17,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:17,100 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/21db5a459f834d16b18f027916db3319 is 50, key is test_row_0/B:col10/1732371915968/Put/seqid=0 2024-11-23T14:25:17,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-23T14:25:17,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741894_1070 (size=12301) 2024-11-23T14:25:17,249 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-23T14:25:17,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:17,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. as already flushing 2024-11-23T14:25:17,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:17,249 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:17,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:17,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:17,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:17,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1732371977280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:17,283 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:17,283 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:17,283 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:17,292 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:17,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34034 deadline: 1732371977290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:17,292 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:17,292 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:17,293 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:17,293 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:17,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34082 deadline: 1732371977290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:17,293 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:17,293 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:17,293 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:17,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:17,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34062 deadline: 1732371977290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:17,293 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:17,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34106 deadline: 1732371977290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:17,297 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:17,297 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:17,297 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:17,300 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:17,300 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:17,301 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:17,403 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-23T14:25:17,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:17,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. as already flushing 2024-11-23T14:25:17,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:17,404 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:17,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:17,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:17,511 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/21db5a459f834d16b18f027916db3319 2024-11-23T14:25:17,543 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/c85a9811fd5841959f31e68d5a073372 is 50, key is test_row_0/C:col10/1732371915968/Put/seqid=0 2024-11-23T14:25:17,560 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-23T14:25:17,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:17,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. as already flushing 2024-11-23T14:25:17,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:17,560 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:17,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:17,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:17,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741895_1071 (size=12301) 2024-11-23T14:25:17,591 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/c85a9811fd5841959f31e68d5a073372 2024-11-23T14:25:17,601 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/1e9fdc3213164ee9a757f27953fd6c3e as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/1e9fdc3213164ee9a757f27953fd6c3e 2024-11-23T14:25:17,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-23T14:25:17,611 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/1e9fdc3213164ee9a757f27953fd6c3e, entries=150, sequenceid=277, filesize=12.0 K 2024-11-23T14:25:17,612 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/21db5a459f834d16b18f027916db3319 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/21db5a459f834d16b18f027916db3319 2024-11-23T14:25:17,621 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/21db5a459f834d16b18f027916db3319, entries=150, sequenceid=277, filesize=12.0 K 2024-11-23T14:25:17,622 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/c85a9811fd5841959f31e68d5a073372 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/c85a9811fd5841959f31e68d5a073372 2024-11-23T14:25:17,633 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/c85a9811fd5841959f31e68d5a073372, entries=150, sequenceid=277, filesize=12.0 K 2024-11-23T14:25:17,634 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 5d1382f36edf9c126fa17e1f99c7f3fe in 1027ms, sequenceid=277, compaction requested=true 2024-11-23T14:25:17,635 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5d1382f36edf9c126fa17e1f99c7f3fe: 2024-11-23T14:25:17,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d1382f36edf9c126fa17e1f99c7f3fe:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T14:25:17,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:25:17,635 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-23T14:25:17,635 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-23T14:25:17,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d1382f36edf9c126fa17e1f99c7f3fe:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T14:25:17,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:25:17,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d1382f36edf9c126fa17e1f99c7f3fe:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T14:25:17,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:25:17,636 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-23T14:25:17,636 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-23T14:25:17,636 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. because compaction request was cancelled 2024-11-23T14:25:17,636 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d1382f36edf9c126fa17e1f99c7f3fe:B 2024-11-23T14:25:17,636 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:25:17,637 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-23T14:25:17,637 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-23T14:25:17,637 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. because compaction request was cancelled 2024-11-23T14:25:17,637 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d1382f36edf9c126fa17e1f99c7f3fe:A 2024-11-23T14:25:17,638 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:25:17,638 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1541): 5d1382f36edf9c126fa17e1f99c7f3fe/C is initiating minor compaction (all files) 2024-11-23T14:25:17,638 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d1382f36edf9c126fa17e1f99c7f3fe/C in TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:17,638 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/f26f8853a26e452289bd01f69b5d29c3, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/d5656e6d9746440497424aedd94656d5, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/c85a9811fd5841959f31e68d5a073372] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp, totalSize=36.3 K 2024-11-23T14:25:17,639 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting f26f8853a26e452289bd01f69b5d29c3, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732371914625 2024-11-23T14:25:17,640 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting d5656e6d9746440497424aedd94656d5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732371915816 2024-11-23T14:25:17,640 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting c85a9811fd5841959f31e68d5a073372, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1732371915968 2024-11-23T14:25:17,660 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d1382f36edf9c126fa17e1f99c7f3fe#C#compaction#59 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:25:17,661 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/d1b8d962bea746a0974827bfa5cafbea is 50, key is test_row_0/C:col10/1732371915968/Put/seqid=0 2024-11-23T14:25:17,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741896_1072 (size=12949) 2024-11-23T14:25:17,716 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-23T14:25:17,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:17,716 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 5d1382f36edf9c126fa17e1f99c7f3fe 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-23T14:25:17,725 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/d1b8d962bea746a0974827bfa5cafbea as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/d1b8d962bea746a0974827bfa5cafbea 2024-11-23T14:25:17,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/ecd3b772d7cd4c938f8b754719ad7362 is 50, key is test_row_0/A:col10/1732371916642/Put/seqid=0 2024-11-23T14:25:17,739 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5d1382f36edf9c126fa17e1f99c7f3fe/C of 5d1382f36edf9c126fa17e1f99c7f3fe into d1b8d962bea746a0974827bfa5cafbea(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:25:17,739 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d1382f36edf9c126fa17e1f99c7f3fe: 2024-11-23T14:25:17,739 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., storeName=5d1382f36edf9c126fa17e1f99c7f3fe/C, priority=13, startTime=1732371917635; duration=0sec 2024-11-23T14:25:17,740 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:25:17,740 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d1382f36edf9c126fa17e1f99c7f3fe:C 2024-11-23T14:25:17,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741897_1073 (size=12301) 2024-11-23T14:25:17,771 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/ecd3b772d7cd4c938f8b754719ad7362 2024-11-23T14:25:17,794 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. as already flushing 2024-11-23T14:25:17,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on 5d1382f36edf9c126fa17e1f99c7f3fe 2024-11-23T14:25:17,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/1d841a4d67e14443be84c26c65fc64ae is 50, key is test_row_0/B:col10/1732371916642/Put/seqid=0 2024-11-23T14:25:17,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741898_1074 (size=12301) 2024-11-23T14:25:17,839 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:17,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34082 deadline: 1732371977832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:17,840 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:17,841 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:17,841 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:17,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:17,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34034 deadline: 1732371977836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:17,845 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:17,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:17,848 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:17,849 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:17,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34106 deadline: 1732371977836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:17,851 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:17,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:17,852 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:17,851 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:17,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34062 deadline: 1732371977839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:17,852 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:17,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1732371977842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:17,854 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:17,854 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:17,854 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:17,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:17,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:17,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:17,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:17,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34082 deadline: 1732371977950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:17,953 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:17,953 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:17,953 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:17,969 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:17,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34034 deadline: 1732371977960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:17,970 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:17,970 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:17,970 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:17,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:17,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34106 deadline: 1732371977965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:17,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:17,972 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:17,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:17,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:17,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34062 deadline: 1732371977965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:17,972 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:17,973 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:17,973 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:17,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:17,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1732371977961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:17,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:17,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:17,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:18,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:18,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34082 deadline: 1732371978160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:18,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:18,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:18,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:18,182 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:18,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34034 deadline: 1732371978182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:18,183 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:18,183 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:18,183 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:18,184 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:18,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1732371978182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:18,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:18,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:18,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:18,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:18,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34106 deadline: 1732371978189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:18,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:18,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:18,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:18,197 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:18,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34062 deadline: 1732371978188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:18,198 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:18,198 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:18,198 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:18,224 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/1d841a4d67e14443be84c26c65fc64ae 2024-11-23T14:25:18,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/400fa77350604812a06daf40ca850362 is 50, key is test_row_0/C:col10/1732371916642/Put/seqid=0 2024-11-23T14:25:18,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741899_1075 (size=12301) 2024-11-23T14:25:18,285 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/400fa77350604812a06daf40ca850362 2024-11-23T14:25:18,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/ecd3b772d7cd4c938f8b754719ad7362 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/ecd3b772d7cd4c938f8b754719ad7362 2024-11-23T14:25:18,310 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/ecd3b772d7cd4c938f8b754719ad7362, entries=150, sequenceid=293, filesize=12.0 K 2024-11-23T14:25:18,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/1d841a4d67e14443be84c26c65fc64ae as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/1d841a4d67e14443be84c26c65fc64ae 2024-11-23T14:25:18,328 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/1d841a4d67e14443be84c26c65fc64ae, entries=150, sequenceid=293, filesize=12.0 K 2024-11-23T14:25:18,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/400fa77350604812a06daf40ca850362 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/400fa77350604812a06daf40ca850362 2024-11-23T14:25:18,346 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/400fa77350604812a06daf40ca850362, entries=150, sequenceid=293, filesize=12.0 K 2024-11-23T14:25:18,348 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for 5d1382f36edf9c126fa17e1f99c7f3fe in 632ms, sequenceid=293, compaction requested=true 2024-11-23T14:25:18,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 5d1382f36edf9c126fa17e1f99c7f3fe: 2024-11-23T14:25:18,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:18,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-23T14:25:18,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-23T14:25:18,356 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-23T14:25:18,356 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8780 sec 2024-11-23T14:25:18,360 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 1.8960 sec 2024-11-23T14:25:18,492 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T14:25:18,492 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5d1382f36edf9c126fa17e1f99c7f3fe 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-23T14:25:18,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on 5d1382f36edf9c126fa17e1f99c7f3fe 2024-11-23T14:25:18,516 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/ce831b7a1e9b49bc91b50cd8acaf5111 is 50, key is test_row_0/A:col10/1732371917836/Put/seqid=0 2024-11-23T14:25:18,527 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:18,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34082 deadline: 1732371978513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:18,528 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:18,528 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:18,528 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:18,527 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:18,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34034 deadline: 1732371978519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:18,531 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:18,531 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:18,531 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:18,531 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:18,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34062 deadline: 1732371978524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:18,532 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:18,532 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:18,532 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:18,532 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:18,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1732371978524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:18,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:18,533 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:18,533 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:18,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:18,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34106 deadline: 1732371978528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:18,536 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:18,536 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:18,536 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:18,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741900_1076 (size=14741) 2024-11-23T14:25:18,562 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/ce831b7a1e9b49bc91b50cd8acaf5111 2024-11-23T14:25:18,583 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/0715c60bb3054fbdaacfaa5b2d61529d is 50, key is test_row_0/B:col10/1732371917836/Put/seqid=0 2024-11-23T14:25:18,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741901_1077 (size=12301) 2024-11-23T14:25:18,604 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/0715c60bb3054fbdaacfaa5b2d61529d 2024-11-23T14:25:18,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-23T14:25:18,621 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T14:25:18,622 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-23T14:25:18,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-23T14:25:18,627 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T14:25:18,636 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T14:25:18,636 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T14:25:18,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:18,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-23T14:25:18,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34082 deadline: 1732371978632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:18,638 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:18,638 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:18,638 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:18,648 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/b58122625945402087e47e6300a0e980 is 50, key is test_row_0/C:col10/1732371917836/Put/seqid=0 2024-11-23T14:25:18,648 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:18,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34034 deadline: 1732371978641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:18,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:18,649 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:18,649 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:18,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34062 deadline: 1732371978641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:18,649 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:18,650 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:18,650 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:18,650 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:18,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:18,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1732371978641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:18,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:18,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:18,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:18,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:18,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34106 deadline: 1732371978648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:18,652 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:18,652 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:18,652 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:18,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741902_1078 (size=12301) 2024-11-23T14:25:18,672 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/b58122625945402087e47e6300a0e980 2024-11-23T14:25:18,681 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/ce831b7a1e9b49bc91b50cd8acaf5111 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/ce831b7a1e9b49bc91b50cd8acaf5111 2024-11-23T14:25:18,690 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/ce831b7a1e9b49bc91b50cd8acaf5111, entries=200, sequenceid=318, filesize=14.4 K 2024-11-23T14:25:18,691 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/0715c60bb3054fbdaacfaa5b2d61529d as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/0715c60bb3054fbdaacfaa5b2d61529d 2024-11-23T14:25:18,706 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/0715c60bb3054fbdaacfaa5b2d61529d, entries=150, sequenceid=318, filesize=12.0 K 2024-11-23T14:25:18,708 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/b58122625945402087e47e6300a0e980 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/b58122625945402087e47e6300a0e980 2024-11-23T14:25:18,719 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/b58122625945402087e47e6300a0e980, entries=150, sequenceid=318, filesize=12.0 K 2024-11-23T14:25:18,721 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 5d1382f36edf9c126fa17e1f99c7f3fe in 229ms, sequenceid=318, compaction requested=true 2024-11-23T14:25:18,721 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5d1382f36edf9c126fa17e1f99c7f3fe: 2024-11-23T14:25:18,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d1382f36edf9c126fa17e1f99c7f3fe:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T14:25:18,721 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T14:25:18,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:25:18,722 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T14:25:18,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d1382f36edf9c126fa17e1f99c7f3fe:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T14:25:18,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:25:18,723 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52108 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T14:25:18,723 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1541): 5d1382f36edf9c126fa17e1f99c7f3fe/A is initiating minor compaction (all files) 2024-11-23T14:25:18,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d1382f36edf9c126fa17e1f99c7f3fe:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T14:25:18,723 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d1382f36edf9c126fa17e1f99c7f3fe/A in TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:18,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:25:18,724 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/34163857dc9449ceace9d451fb26d035, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/1e9fdc3213164ee9a757f27953fd6c3e, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/ecd3b772d7cd4c938f8b754719ad7362, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/ce831b7a1e9b49bc91b50cd8acaf5111] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp, totalSize=50.9 K 2024-11-23T14:25:18,724 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49668 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T14:25:18,724 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1541): 5d1382f36edf9c126fa17e1f99c7f3fe/B is initiating minor compaction (all files) 2024-11-23T14:25:18,724 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d1382f36edf9c126fa17e1f99c7f3fe/B in TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:18,724 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/ea84ec993cf14f06a2b46aefdf59d440, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/21db5a459f834d16b18f027916db3319, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/1d841a4d67e14443be84c26c65fc64ae, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/0715c60bb3054fbdaacfaa5b2d61529d] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp, totalSize=48.5 K 2024-11-23T14:25:18,724 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 34163857dc9449ceace9d451fb26d035, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732371915816 2024-11-23T14:25:18,725 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting ea84ec993cf14f06a2b46aefdf59d440, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732371915816 2024-11-23T14:25:18,725 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 21db5a459f834d16b18f027916db3319, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1732371915968 2024-11-23T14:25:18,725 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e9fdc3213164ee9a757f27953fd6c3e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1732371915968 2024-11-23T14:25:18,726 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d841a4d67e14443be84c26c65fc64ae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1732371916619 2024-11-23T14:25:18,726 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting ecd3b772d7cd4c938f8b754719ad7362, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1732371916619 2024-11-23T14:25:18,726 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce831b7a1e9b49bc91b50cd8acaf5111, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1732371917828 2024-11-23T14:25:18,726 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 0715c60bb3054fbdaacfaa5b2d61529d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1732371917828 2024-11-23T14:25:18,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-23T14:25:18,756 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d1382f36edf9c126fa17e1f99c7f3fe#A#compaction#66 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:25:18,757 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/48c220a788e246b6bda49c92d1ae3935 is 50, key is test_row_0/A:col10/1732371917836/Put/seqid=0 2024-11-23T14:25:18,771 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d1382f36edf9c126fa17e1f99c7f3fe#B#compaction#67 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:25:18,771 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/f7e3f2529ae04c09a4e39818d31df6e1 is 50, key is test_row_0/B:col10/1732371917836/Put/seqid=0 2024-11-23T14:25:18,791 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-23T14:25:18,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:18,793 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 5d1382f36edf9c126fa17e1f99c7f3fe 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-23T14:25:18,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741903_1079 (size=13051) 2024-11-23T14:25:18,821 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/48c220a788e246b6bda49c92d1ae3935 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/48c220a788e246b6bda49c92d1ae3935 2024-11-23T14:25:18,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/dfa2c4c4ae29490eb65becaf574f3087 is 50, key is test_row_0/A:col10/1732371918524/Put/seqid=0 2024-11-23T14:25:18,835 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5d1382f36edf9c126fa17e1f99c7f3fe/A of 5d1382f36edf9c126fa17e1f99c7f3fe into 48c220a788e246b6bda49c92d1ae3935(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:25:18,835 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d1382f36edf9c126fa17e1f99c7f3fe: 2024-11-23T14:25:18,835 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., storeName=5d1382f36edf9c126fa17e1f99c7f3fe/A, priority=12, startTime=1732371918721; duration=0sec 2024-11-23T14:25:18,835 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:25:18,835 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d1382f36edf9c126fa17e1f99c7f3fe:A 2024-11-23T14:25:18,836 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:25:18,837 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:25:18,837 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1541): 5d1382f36edf9c126fa17e1f99c7f3fe/C is initiating minor compaction (all files) 2024-11-23T14:25:18,837 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d1382f36edf9c126fa17e1f99c7f3fe/C in TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:18,838 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/d1b8d962bea746a0974827bfa5cafbea, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/400fa77350604812a06daf40ca850362, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/b58122625945402087e47e6300a0e980] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp, totalSize=36.7 K 2024-11-23T14:25:18,838 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting d1b8d962bea746a0974827bfa5cafbea, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1732371915968 2024-11-23T14:25:18,839 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 400fa77350604812a06daf40ca850362, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1732371916619 2024-11-23T14:25:18,839 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting b58122625945402087e47e6300a0e980, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1732371917828 2024-11-23T14:25:18,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741904_1080 (size=13051) 2024-11-23T14:25:18,857 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. as already flushing 2024-11-23T14:25:18,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on 5d1382f36edf9c126fa17e1f99c7f3fe 2024-11-23T14:25:18,864 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/f7e3f2529ae04c09a4e39818d31df6e1 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/f7e3f2529ae04c09a4e39818d31df6e1 2024-11-23T14:25:18,874 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5d1382f36edf9c126fa17e1f99c7f3fe/B of 5d1382f36edf9c126fa17e1f99c7f3fe into f7e3f2529ae04c09a4e39818d31df6e1(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:25:18,875 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d1382f36edf9c126fa17e1f99c7f3fe: 2024-11-23T14:25:18,875 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., storeName=5d1382f36edf9c126fa17e1f99c7f3fe/B, priority=12, startTime=1732371918722; duration=0sec 2024-11-23T14:25:18,875 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:25:18,875 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d1382f36edf9c126fa17e1f99c7f3fe:B 2024-11-23T14:25:18,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741905_1081 (size=12301) 2024-11-23T14:25:18,891 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d1382f36edf9c126fa17e1f99c7f3fe#C#compaction#69 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:25:18,892 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/06455c05b7d045aa9b94b58a3acf1a3a is 50, key is test_row_0/C:col10/1732371917836/Put/seqid=0 2024-11-23T14:25:18,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741906_1082 (size=13051) 2024-11-23T14:25:18,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:18,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34062 deadline: 1732371978944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:18,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:18,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34082 deadline: 1732371978943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:18,957 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:18,957 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:18,957 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:18,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:18,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:18,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:18,958 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:18,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1732371978951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:18,959 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:18,959 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:18,959 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:18,959 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:18,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34034 deadline: 1732371978954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:18,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-23T14:25:18,960 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:18,960 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:18,960 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:18,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:18,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34106 deadline: 1732371978956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:18,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:18,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:18,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:19,061 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:19,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34062 deadline: 1732371979060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:19,061 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:19,061 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:19,062 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:19,061 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:19,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34082 deadline: 1732371979060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:19,062 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:19,062 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:19,062 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:19,074 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:19,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1732371979071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:19,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:19,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34034 deadline: 1732371979071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:19,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:19,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34106 deadline: 1732371979071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:19,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:19,075 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:19,076 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:19,076 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:19,076 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:19,076 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:19,076 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:19,076 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:19,076 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:19,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-23T14:25:19,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:19,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34082 deadline: 1732371979271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:19,272 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:19,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:19,272 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:19,272 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:19,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34062 deadline: 1732371979271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:19,273 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:19,273 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:19,273 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:19,282 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/dfa2c4c4ae29490eb65becaf574f3087 2024-11-23T14:25:19,283 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:19,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34034 deadline: 1732371979281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:19,284 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:19,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:19,284 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:19,284 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:19,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1732371979281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:19,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:19,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:19,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:19,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:19,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34106 deadline: 1732371979281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:19,285 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:19,285 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:19,285 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:19,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/fca49e1457244240b9226612c74a615e is 50, key is test_row_0/B:col10/1732371918524/Put/seqid=0 2024-11-23T14:25:19,334 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/06455c05b7d045aa9b94b58a3acf1a3a as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/06455c05b7d045aa9b94b58a3acf1a3a 2024-11-23T14:25:19,345 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5d1382f36edf9c126fa17e1f99c7f3fe/C of 5d1382f36edf9c126fa17e1f99c7f3fe into 06455c05b7d045aa9b94b58a3acf1a3a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:25:19,345 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d1382f36edf9c126fa17e1f99c7f3fe: 2024-11-23T14:25:19,345 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., storeName=5d1382f36edf9c126fa17e1f99c7f3fe/C, priority=13, startTime=1732371918723; duration=0sec 2024-11-23T14:25:19,346 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:25:19,346 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d1382f36edf9c126fa17e1f99c7f3fe:C 2024-11-23T14:25:19,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741907_1083 (size=12301) 2024-11-23T14:25:19,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:19,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34082 deadline: 1732371979580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:19,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:19,582 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:19,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34062 deadline: 1732371979580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:19,583 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:19,583 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:19,583 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:19,583 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:19,583 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:19,590 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:19,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34034 deadline: 1732371979590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:19,591 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:19,591 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:19,591 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:19,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:19,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1732371979590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:19,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:19,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:19,593 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:19,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:19,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34106 deadline: 1732371979590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:19,596 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:19,596 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:19,596 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:19,746 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/fca49e1457244240b9226612c74a615e 2024-11-23T14:25:19,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/b1679cc41af34aa3a962fa5ae5252479 is 50, key is test_row_0/C:col10/1732371918524/Put/seqid=0 2024-11-23T14:25:19,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741908_1084 (size=12301) 2024-11-23T14:25:19,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-23T14:25:20,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:20,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34082 deadline: 1732371980090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:20,091 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:20,091 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:20,091 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:20,094 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:20,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34062 deadline: 1732371980090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:20,095 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:20,095 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:20,095 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:20,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:20,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1732371980100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:20,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:20,104 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:20,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34034 deadline: 1732371980100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:20,105 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:20,105 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:20,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:20,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:20,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:20,110 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:20,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34106 deadline: 1732371980110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:20,111 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:20,111 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:20,111 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:20,166 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/b1679cc41af34aa3a962fa5ae5252479 2024-11-23T14:25:20,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/dfa2c4c4ae29490eb65becaf574f3087 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/dfa2c4c4ae29490eb65becaf574f3087 2024-11-23T14:25:20,180 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/dfa2c4c4ae29490eb65becaf574f3087, entries=150, sequenceid=330, filesize=12.0 K 2024-11-23T14:25:20,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/fca49e1457244240b9226612c74a615e as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/fca49e1457244240b9226612c74a615e 2024-11-23T14:25:20,189 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/fca49e1457244240b9226612c74a615e, entries=150, sequenceid=330, filesize=12.0 K 2024-11-23T14:25:20,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/b1679cc41af34aa3a962fa5ae5252479 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/b1679cc41af34aa3a962fa5ae5252479 2024-11-23T14:25:20,205 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/b1679cc41af34aa3a962fa5ae5252479, entries=150, sequenceid=330, filesize=12.0 K 2024-11-23T14:25:20,206 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 5d1382f36edf9c126fa17e1f99c7f3fe in 1413ms, sequenceid=330, compaction requested=false 2024-11-23T14:25:20,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 5d1382f36edf9c126fa17e1f99c7f3fe: 2024-11-23T14:25:20,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:20,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-23T14:25:20,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-23T14:25:20,212 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-23T14:25:20,212 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5730 sec 2024-11-23T14:25:20,215 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.5910 sec 2024-11-23T14:25:20,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-23T14:25:20,791 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T14:25:20,792 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-23T14:25:20,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-23T14:25:20,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-23T14:25:20,799 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T14:25:20,800 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T14:25:20,800 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T14:25:20,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-23T14:25:20,964 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-23T14:25:20,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:20,965 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 5d1382f36edf9c126fa17e1f99c7f3fe 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-23T14:25:20,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/7f09fd65de4844888f4a9d9055ea805a is 50, key is test_row_0/A:col10/1732371918953/Put/seqid=0 2024-11-23T14:25:21,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741909_1085 (size=12301) 2024-11-23T14:25:21,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on 5d1382f36edf9c126fa17e1f99c7f3fe 2024-11-23T14:25:21,103 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. as already flushing 2024-11-23T14:25:21,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-23T14:25:21,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:21,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34106 deadline: 1732371981127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:21,137 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:21,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34062 deadline: 1732371981132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:21,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:21,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34082 deadline: 1732371981132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:21,139 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:21,139 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:21,139 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:21,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:21,139 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:21,139 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:21,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34034 deadline: 1732371981134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:21,139 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:21,140 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:21,140 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:21,140 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:21,142 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to c6f7b1897123,35385,1732371902767 for 'test_row_1' in 5d1382f36edf9c126fa17e1f99c7f3fe of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-23T14:25:21,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:21,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:21,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:21,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:21,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1732371981134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:21,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:21,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:21,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:21,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:21,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:21,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34034 deadline: 1732371981251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:21,253 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:21,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34062 deadline: 1732371981251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:21,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34082 deadline: 1732371981251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:21,253 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:21,253 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:21,253 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:21,253 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:21,253 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:21,253 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:21,253 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:21,253 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:21,253 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:21,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:21,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1732371981260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:21,262 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:21,262 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:21,262 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:21,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-23T14:25:21,427 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/7f09fd65de4844888f4a9d9055ea805a 2024-11-23T14:25:21,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/2142bf98f443484689941c1b3fa19bba is 50, key is test_row_0/B:col10/1732371918953/Put/seqid=0 2024-11-23T14:25:21,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741910_1086 (size=12301) 2024-11-23T14:25:21,442 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/2142bf98f443484689941c1b3fa19bba 2024-11-23T14:25:21,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:21,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34062 deadline: 1732371981460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:21,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:21,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34034 deadline: 1732371981460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:21,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:21,462 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:21,462 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:21,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34082 deadline: 1732371981460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:21,462 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:21,462 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:21,462 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:21,462 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:21,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/8ce659ccc96245c79383fac2f0a30f20 is 50, key is test_row_0/C:col10/1732371918953/Put/seqid=0 2024-11-23T14:25:21,463 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:21,463 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:21,463 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:21,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741911_1087 (size=12301) 2024-11-23T14:25:21,472 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/8ce659ccc96245c79383fac2f0a30f20 2024-11-23T14:25:21,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:21,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1732371981471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:21,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:21,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:21,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:21,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/7f09fd65de4844888f4a9d9055ea805a as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/7f09fd65de4844888f4a9d9055ea805a 2024-11-23T14:25:21,486 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/7f09fd65de4844888f4a9d9055ea805a, entries=150, sequenceid=357, filesize=12.0 K 2024-11-23T14:25:21,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/2142bf98f443484689941c1b3fa19bba as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/2142bf98f443484689941c1b3fa19bba 2024-11-23T14:25:21,495 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/2142bf98f443484689941c1b3fa19bba, entries=150, sequenceid=357, filesize=12.0 K 2024-11-23T14:25:21,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/8ce659ccc96245c79383fac2f0a30f20 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/8ce659ccc96245c79383fac2f0a30f20 2024-11-23T14:25:21,502 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/8ce659ccc96245c79383fac2f0a30f20, entries=150, sequenceid=357, filesize=12.0 K 2024-11-23T14:25:21,503 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for 5d1382f36edf9c126fa17e1f99c7f3fe in 539ms, sequenceid=357, compaction requested=true 2024-11-23T14:25:21,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 5d1382f36edf9c126fa17e1f99c7f3fe: 2024-11-23T14:25:21,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:21,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-23T14:25:21,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-23T14:25:21,506 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-23T14:25:21,507 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 705 msec 2024-11-23T14:25:21,509 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 715 msec 2024-11-23T14:25:21,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on 5d1382f36edf9c126fa17e1f99c7f3fe 2024-11-23T14:25:21,775 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T14:25:21,775 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5d1382f36edf9c126fa17e1f99c7f3fe 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-23T14:25:21,791 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/710d99614bd847bea626db0412f8abd2 is 50, key is test_row_0/A:col10/1732371921773/Put/seqid=0 2024-11-23T14:25:21,833 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:21,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34062 deadline: 1732371981824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:21,834 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:21,834 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:21,834 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:21,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741912_1088 (size=14741) 2024-11-23T14:25:21,836 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/710d99614bd847bea626db0412f8abd2 2024-11-23T14:25:21,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:21,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34034 deadline: 1732371981833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:21,840 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:21,840 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:21,840 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:21,840 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:21,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1732371981833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:21,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:21,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:21,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:21,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:21,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34082 deadline: 1732371981833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:21,842 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:21,843 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:21,843 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:21,848 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/36bdc8aaa0a54d2197f056b414ac6761 is 50, key is test_row_0/B:col10/1732371921773/Put/seqid=0 2024-11-23T14:25:21,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741913_1089 (size=12301) 2024-11-23T14:25:21,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-23T14:25:21,930 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T14:25:21,931 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-23T14:25:21,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-23T14:25:21,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-23T14:25:21,939 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T14:25:21,940 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T14:25:21,941 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T14:25:21,950 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:21,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34034 deadline: 1732371981950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:21,951 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:21,951 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:21,951 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:21,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:21,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34062 deadline: 1732371981940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:21,953 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:21,953 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:21,953 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:21,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:21,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1732371981955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:21,959 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:21,959 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:21,959 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:21,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:21,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34082 deadline: 1732371981955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:21,959 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:21,959 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:21,960 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:22,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-23T14:25:22,095 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-23T14:25:22,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:22,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. as already flushing 2024-11-23T14:25:22,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:22,096 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:22,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:22,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:22,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:22,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34034 deadline: 1732371982160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:22,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:22,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34062 deadline: 1732371982160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:22,161 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:22,161 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:22,161 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:22,161 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:22,161 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:22,162 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:22,171 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:22,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1732371982170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:22,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:22,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:22,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:22,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34082 deadline: 1732371982170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:22,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:22,172 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:22,172 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:22,172 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:22,249 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-23T14:25:22,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:22,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. as already flushing 2024-11-23T14:25:22,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:22,250 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:22,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:22,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:22,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-23T14:25:22,260 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/36bdc8aaa0a54d2197f056b414ac6761 2024-11-23T14:25:22,272 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/aec993c358554e5a8781d73497d17c9d is 50, key is test_row_0/C:col10/1732371921773/Put/seqid=0 2024-11-23T14:25:22,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741914_1090 (size=12301) 2024-11-23T14:25:22,288 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/aec993c358554e5a8781d73497d17c9d 2024-11-23T14:25:22,304 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/710d99614bd847bea626db0412f8abd2 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/710d99614bd847bea626db0412f8abd2 2024-11-23T14:25:22,312 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/710d99614bd847bea626db0412f8abd2, entries=200, sequenceid=371, filesize=14.4 K 2024-11-23T14:25:22,317 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/36bdc8aaa0a54d2197f056b414ac6761 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/36bdc8aaa0a54d2197f056b414ac6761 2024-11-23T14:25:22,325 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/36bdc8aaa0a54d2197f056b414ac6761, entries=150, sequenceid=371, filesize=12.0 K 2024-11-23T14:25:22,327 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/aec993c358554e5a8781d73497d17c9d as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/aec993c358554e5a8781d73497d17c9d 2024-11-23T14:25:22,345 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/aec993c358554e5a8781d73497d17c9d, entries=150, sequenceid=371, filesize=12.0 K 2024-11-23T14:25:22,349 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for 5d1382f36edf9c126fa17e1f99c7f3fe in 574ms, sequenceid=371, compaction requested=true 2024-11-23T14:25:22,349 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5d1382f36edf9c126fa17e1f99c7f3fe: 2024-11-23T14:25:22,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d1382f36edf9c126fa17e1f99c7f3fe:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T14:25:22,349 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T14:25:22,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:25:22,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d1382f36edf9c126fa17e1f99c7f3fe:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T14:25:22,349 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T14:25:22,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:25:22,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d1382f36edf9c126fa17e1f99c7f3fe:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T14:25:22,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:25:22,351 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T14:25:22,351 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52394 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T14:25:22,351 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1541): 5d1382f36edf9c126fa17e1f99c7f3fe/A is initiating minor compaction (all files) 2024-11-23T14:25:22,351 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1541): 5d1382f36edf9c126fa17e1f99c7f3fe/B is initiating minor compaction (all files) 2024-11-23T14:25:22,351 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d1382f36edf9c126fa17e1f99c7f3fe/B in TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:22,351 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d1382f36edf9c126fa17e1f99c7f3fe/A in TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:22,351 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/48c220a788e246b6bda49c92d1ae3935, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/dfa2c4c4ae29490eb65becaf574f3087, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/7f09fd65de4844888f4a9d9055ea805a, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/710d99614bd847bea626db0412f8abd2] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp, totalSize=51.2 K 2024-11-23T14:25:22,351 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/f7e3f2529ae04c09a4e39818d31df6e1, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/fca49e1457244240b9226612c74a615e, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/2142bf98f443484689941c1b3fa19bba, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/36bdc8aaa0a54d2197f056b414ac6761] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp, totalSize=48.8 K 2024-11-23T14:25:22,352 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting f7e3f2529ae04c09a4e39818d31df6e1, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1732371917828 2024-11-23T14:25:22,352 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48c220a788e246b6bda49c92d1ae3935, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1732371917828 2024-11-23T14:25:22,352 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting fca49e1457244240b9226612c74a615e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1732371918508 2024-11-23T14:25:22,353 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 2142bf98f443484689941c1b3fa19bba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1732371918919 2024-11-23T14:25:22,355 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 36bdc8aaa0a54d2197f056b414ac6761, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1732371921128 2024-11-23T14:25:22,355 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting dfa2c4c4ae29490eb65becaf574f3087, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1732371918508 2024-11-23T14:25:22,356 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f09fd65de4844888f4a9d9055ea805a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1732371918919 2024-11-23T14:25:22,357 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 710d99614bd847bea626db0412f8abd2, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1732371921126 2024-11-23T14:25:22,378 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d1382f36edf9c126fa17e1f99c7f3fe#B#compaction#78 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:25:22,381 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/3971d71c698e41acab7a199402c6aeff is 50, key is test_row_0/B:col10/1732371921773/Put/seqid=0 2024-11-23T14:25:22,382 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d1382f36edf9c126fa17e1f99c7f3fe#A#compaction#79 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:25:22,382 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/49ad3fbc31e44f19bf88c47d8f0dcaf9 is 50, key is test_row_0/A:col10/1732371921773/Put/seqid=0 2024-11-23T14:25:22,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741915_1091 (size=13187) 2024-11-23T14:25:22,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741916_1092 (size=13187) 2024-11-23T14:25:22,403 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-23T14:25:22,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:22,404 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 5d1382f36edf9c126fa17e1f99c7f3fe 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-23T14:25:22,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/594d187c4ea743e48263be15095bfdce is 50, key is test_row_0/A:col10/1732371921831/Put/seqid=0 2024-11-23T14:25:22,416 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/3971d71c698e41acab7a199402c6aeff as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/3971d71c698e41acab7a199402c6aeff 2024-11-23T14:25:22,434 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5d1382f36edf9c126fa17e1f99c7f3fe/B of 5d1382f36edf9c126fa17e1f99c7f3fe into 3971d71c698e41acab7a199402c6aeff(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:25:22,434 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d1382f36edf9c126fa17e1f99c7f3fe: 2024-11-23T14:25:22,434 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/49ad3fbc31e44f19bf88c47d8f0dcaf9 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/49ad3fbc31e44f19bf88c47d8f0dcaf9 2024-11-23T14:25:22,434 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., storeName=5d1382f36edf9c126fa17e1f99c7f3fe/B, priority=12, startTime=1732371922349; duration=0sec 2024-11-23T14:25:22,434 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:25:22,434 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d1382f36edf9c126fa17e1f99c7f3fe:B 2024-11-23T14:25:22,434 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T14:25:22,449 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T14:25:22,449 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1541): 5d1382f36edf9c126fa17e1f99c7f3fe/C is initiating minor compaction (all files) 2024-11-23T14:25:22,449 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d1382f36edf9c126fa17e1f99c7f3fe/C in TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:22,449 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/06455c05b7d045aa9b94b58a3acf1a3a, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/b1679cc41af34aa3a962fa5ae5252479, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/8ce659ccc96245c79383fac2f0a30f20, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/aec993c358554e5a8781d73497d17c9d] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp, totalSize=48.8 K 2024-11-23T14:25:22,452 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 06455c05b7d045aa9b94b58a3acf1a3a, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1732371917828 2024-11-23T14:25:22,453 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting b1679cc41af34aa3a962fa5ae5252479, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1732371918508 2024-11-23T14:25:22,454 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ce659ccc96245c79383fac2f0a30f20, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1732371918919 2024-11-23T14:25:22,455 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting aec993c358554e5a8781d73497d17c9d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1732371921128 2024-11-23T14:25:22,457 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5d1382f36edf9c126fa17e1f99c7f3fe/A of 5d1382f36edf9c126fa17e1f99c7f3fe into 49ad3fbc31e44f19bf88c47d8f0dcaf9(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:25:22,457 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d1382f36edf9c126fa17e1f99c7f3fe: 2024-11-23T14:25:22,457 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., storeName=5d1382f36edf9c126fa17e1f99c7f3fe/A, priority=12, startTime=1732371922349; duration=0sec 2024-11-23T14:25:22,457 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:25:22,458 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d1382f36edf9c126fa17e1f99c7f3fe:A 2024-11-23T14:25:22,478 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. as already flushing 2024-11-23T14:25:22,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on 5d1382f36edf9c126fa17e1f99c7f3fe 2024-11-23T14:25:22,481 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d1382f36edf9c126fa17e1f99c7f3fe#C#compaction#81 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:25:22,482 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/e9870db655984b7ba06bb5c72d4e08a8 is 50, key is test_row_0/C:col10/1732371921773/Put/seqid=0 2024-11-23T14:25:22,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741917_1093 (size=12301) 2024-11-23T14:25:22,483 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/594d187c4ea743e48263be15095bfdce 2024-11-23T14:25:22,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/8fab80edcc3d4988b669c0c06e4cad6e is 50, key is test_row_0/B:col10/1732371921831/Put/seqid=0 2024-11-23T14:25:22,508 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:22,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34062 deadline: 1732371982501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:22,509 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:22,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34034 deadline: 1732371982503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:22,509 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:22,509 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:22,509 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:22,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:22,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1732371982503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:22,511 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:22,511 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:22,511 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:22,511 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:22,511 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:22,511 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:22,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:22,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34082 deadline: 1732371982512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:22,515 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:22,515 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:22,515 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:22,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741918_1094 (size=13187) 2024-11-23T14:25:22,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741919_1095 (size=12301) 2024-11-23T14:25:22,543 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/e9870db655984b7ba06bb5c72d4e08a8 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/e9870db655984b7ba06bb5c72d4e08a8 2024-11-23T14:25:22,552 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5d1382f36edf9c126fa17e1f99c7f3fe/C of 5d1382f36edf9c126fa17e1f99c7f3fe into e9870db655984b7ba06bb5c72d4e08a8(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:25:22,552 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d1382f36edf9c126fa17e1f99c7f3fe: 2024-11-23T14:25:22,552 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., storeName=5d1382f36edf9c126fa17e1f99c7f3fe/C, priority=12, startTime=1732371922349; duration=0sec 2024-11-23T14:25:22,552 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:25:22,552 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d1382f36edf9c126fa17e1f99c7f3fe:C 2024-11-23T14:25:22,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-23T14:25:22,621 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:22,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34034 deadline: 1732371982621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:22,622 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:22,622 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:22,622 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:22,622 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:22,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1732371982621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:22,623 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:22,623 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:22,623 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:22,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:22,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34082 deadline: 1732371982621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:22,624 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:22,624 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:22,624 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:22,624 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:22,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34062 deadline: 1732371982621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:22,625 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:22,625 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:22,625 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:22,832 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:22,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34034 deadline: 1732371982830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:22,832 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:22,832 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:22,833 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:22,833 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:22,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1732371982830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:22,833 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:22,833 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:22,833 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:22,833 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:22,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34082 deadline: 1732371982830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:22,833 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:22,833 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:22,833 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:22,833 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:22,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34062 deadline: 1732371982831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:22,834 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:22,834 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:22,834 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:22,932 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/8fab80edcc3d4988b669c0c06e4cad6e 2024-11-23T14:25:22,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/d22071479c8d4828b17c68941c6cf521 is 50, key is test_row_0/C:col10/1732371921831/Put/seqid=0 2024-11-23T14:25:23,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741920_1096 (size=12301) 2024-11-23T14:25:23,018 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/d22071479c8d4828b17c68941c6cf521 2024-11-23T14:25:23,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/594d187c4ea743e48263be15095bfdce as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/594d187c4ea743e48263be15095bfdce 2024-11-23T14:25:23,038 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/594d187c4ea743e48263be15095bfdce, entries=150, sequenceid=394, filesize=12.0 K 2024-11-23T14:25:23,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/8fab80edcc3d4988b669c0c06e4cad6e as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/8fab80edcc3d4988b669c0c06e4cad6e 2024-11-23T14:25:23,049 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/8fab80edcc3d4988b669c0c06e4cad6e, entries=150, sequenceid=394, filesize=12.0 K 2024-11-23T14:25:23,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/d22071479c8d4828b17c68941c6cf521 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/d22071479c8d4828b17c68941c6cf521 2024-11-23T14:25:23,058 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/d22071479c8d4828b17c68941c6cf521, entries=150, sequenceid=394, filesize=12.0 K 2024-11-23T14:25:23,060 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 5d1382f36edf9c126fa17e1f99c7f3fe in 657ms, sequenceid=394, compaction requested=false 2024-11-23T14:25:23,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 5d1382f36edf9c126fa17e1f99c7f3fe: 2024-11-23T14:25:23,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:23,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-23T14:25:23,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-23T14:25:23,065 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-23T14:25:23,065 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1220 sec 2024-11-23T14:25:23,068 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 1.1350 sec 2024-11-23T14:25:23,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-23T14:25:23,081 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T14:25:23,082 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-23T14:25:23,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-23T14:25:23,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-23T14:25:23,086 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T14:25:23,088 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T14:25:23,088 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T14:25:23,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on 5d1382f36edf9c126fa17e1f99c7f3fe 2024-11-23T14:25:23,145 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T14:25:23,146 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5d1382f36edf9c126fa17e1f99c7f3fe 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-23T14:25:23,162 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/13362274105c4039a1c2457f499e8bc4 is 50, key is test_row_0/A:col10/1732371923144/Put/seqid=0 2024-11-23T14:25:23,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-23T14:25:23,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:23,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:23,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1732371983184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:23,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34062 deadline: 1732371983186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:23,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:23,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:23,198 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:23,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:23,198 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:23,198 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:23,198 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:23,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34106 deadline: 1732371983188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:23,198 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:23,198 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:23,198 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:23,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741921_1097 (size=14741) 2024-11-23T14:25:23,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:23,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34034 deadline: 1732371983197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:23,205 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=411 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/13362274105c4039a1c2457f499e8bc4 2024-11-23T14:25:23,208 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:23,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34082 deadline: 1732371983197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:23,209 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:23,209 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:23,209 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:23,220 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:23,220 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:23,220 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:23,232 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/7f07a212d1a848568096f1af7f97846f is 50, key is test_row_0/B:col10/1732371923144/Put/seqid=0 2024-11-23T14:25:23,244 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-23T14:25:23,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:23,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. as already flushing 2024-11-23T14:25:23,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:23,244 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:23,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:23,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:23,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741922_1098 (size=12301) 2024-11-23T14:25:23,304 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:23,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1732371983301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:23,304 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:23,304 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:23,304 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:23,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:23,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34062 deadline: 1732371983301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:23,304 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:23,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34106 deadline: 1732371983301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:23,305 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:23,306 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:23,306 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:23,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:23,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:23,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:23,323 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:23,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34082 deadline: 1732371983321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:23,325 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:23,325 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:23,325 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:23,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:23,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34034 deadline: 1732371983330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:23,334 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:23,334 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:23,334 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:23,398 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-23T14:25:23,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:23,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. as already flushing 2024-11-23T14:25:23,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:23,399 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:23,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-23T14:25:23,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:23,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:23,512 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:23,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:23,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1732371983510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:23,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34062 deadline: 1732371983510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:23,513 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:23,513 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:23,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:23,513 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:23,513 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:23,513 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:23,513 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:23,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34106 deadline: 1732371983510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:23,514 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:23,514 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:23,514 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:23,532 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:23,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34082 deadline: 1732371983530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:23,533 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:23,533 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:23,533 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:23,542 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:23,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34034 deadline: 1732371983540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:23,543 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:23,543 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:23,543 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:23,558 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-23T14:25:23,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:23,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. as already flushing 2024-11-23T14:25:23,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:23,559 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:23,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:23,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:23,676 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=411 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/7f07a212d1a848568096f1af7f97846f 2024-11-23T14:25:23,685 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/99db463312e841d091d23483579fd35e is 50, key is test_row_0/C:col10/1732371923144/Put/seqid=0 2024-11-23T14:25:23,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741923_1099 (size=12301) 2024-11-23T14:25:23,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-23T14:25:23,712 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-23T14:25:23,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:23,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. as already flushing 2024-11-23T14:25:23,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:23,712 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:23,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:23,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:23,821 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:23,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34062 deadline: 1732371983820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:23,822 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:23,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1732371983820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:23,822 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:23,822 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:23,822 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:23,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:23,822 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:23,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:23,823 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:23,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34106 deadline: 1732371983820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:23,823 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:23,823 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:23,823 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:23,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:23,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34082 deadline: 1732371983840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:23,842 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:23,842 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:23,842 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:23,850 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:23,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34034 deadline: 1732371983850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:23,851 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:23,851 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:23,851 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:23,865 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-23T14:25:23,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:23,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. as already flushing 2024-11-23T14:25:23,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:23,865 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:23,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:23,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:24,033 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-23T14:25:24,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:24,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. as already flushing 2024-11-23T14:25:24,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:24,033 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:24,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:24,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:24,095 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=411 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/99db463312e841d091d23483579fd35e 2024-11-23T14:25:24,106 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/13362274105c4039a1c2457f499e8bc4 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/13362274105c4039a1c2457f499e8bc4 2024-11-23T14:25:24,115 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/13362274105c4039a1c2457f499e8bc4, entries=200, sequenceid=411, filesize=14.4 K 2024-11-23T14:25:24,117 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/7f07a212d1a848568096f1af7f97846f as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/7f07a212d1a848568096f1af7f97846f 2024-11-23T14:25:24,125 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/7f07a212d1a848568096f1af7f97846f, entries=150, sequenceid=411, filesize=12.0 K 2024-11-23T14:25:24,128 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/99db463312e841d091d23483579fd35e as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/99db463312e841d091d23483579fd35e 2024-11-23T14:25:24,141 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/99db463312e841d091d23483579fd35e, entries=150, sequenceid=411, filesize=12.0 K 2024-11-23T14:25:24,142 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 5d1382f36edf9c126fa17e1f99c7f3fe in 997ms, sequenceid=411, compaction requested=true 2024-11-23T14:25:24,142 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5d1382f36edf9c126fa17e1f99c7f3fe: 2024-11-23T14:25:24,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d1382f36edf9c126fa17e1f99c7f3fe:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T14:25:24,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:25:24,142 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:25:24,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d1382f36edf9c126fa17e1f99c7f3fe:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T14:25:24,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:25:24,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d1382f36edf9c126fa17e1f99c7f3fe:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T14:25:24,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-23T14:25:24,143 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:25:24,144 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40229 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:25:24,144 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1541): 5d1382f36edf9c126fa17e1f99c7f3fe/A is initiating minor compaction (all files) 2024-11-23T14:25:24,144 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:25:24,144 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d1382f36edf9c126fa17e1f99c7f3fe/A in TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:24,144 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1541): 5d1382f36edf9c126fa17e1f99c7f3fe/B is initiating minor compaction (all files) 2024-11-23T14:25:24,144 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d1382f36edf9c126fa17e1f99c7f3fe/B in TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:24,145 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/49ad3fbc31e44f19bf88c47d8f0dcaf9, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/594d187c4ea743e48263be15095bfdce, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/13362274105c4039a1c2457f499e8bc4] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp, totalSize=39.3 K 2024-11-23T14:25:24,145 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/3971d71c698e41acab7a199402c6aeff, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/8fab80edcc3d4988b669c0c06e4cad6e, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/7f07a212d1a848568096f1af7f97846f] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp, totalSize=36.9 K 2024-11-23T14:25:24,145 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49ad3fbc31e44f19bf88c47d8f0dcaf9, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1732371921128 2024-11-23T14:25:24,145 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 3971d71c698e41acab7a199402c6aeff, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1732371921128 2024-11-23T14:25:24,145 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 8fab80edcc3d4988b669c0c06e4cad6e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1732371921823 2024-11-23T14:25:24,145 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 594d187c4ea743e48263be15095bfdce, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1732371921823 2024-11-23T14:25:24,146 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f07a212d1a848568096f1af7f97846f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1732371922476 2024-11-23T14:25:24,146 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 13362274105c4039a1c2457f499e8bc4, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1732371922476 2024-11-23T14:25:24,166 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d1382f36edf9c126fa17e1f99c7f3fe#A#compaction#87 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:25:24,166 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d1382f36edf9c126fa17e1f99c7f3fe#B#compaction#88 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:25:24,166 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/d7fb76d3b4264c7998dbc2566cf97835 is 50, key is test_row_0/A:col10/1732371923144/Put/seqid=0 2024-11-23T14:25:24,167 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/e998814dd4f84b939003d89c5c1ed246 is 50, key is test_row_0/B:col10/1732371923144/Put/seqid=0 2024-11-23T14:25:24,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741925_1101 (size=13289) 2024-11-23T14:25:24,187 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-23T14:25:24,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741924_1100 (size=13289) 2024-11-23T14:25:24,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:24,188 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing 5d1382f36edf9c126fa17e1f99c7f3fe 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-23T14:25:24,190 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/e998814dd4f84b939003d89c5c1ed246 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/e998814dd4f84b939003d89c5c1ed246 2024-11-23T14:25:24,199 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5d1382f36edf9c126fa17e1f99c7f3fe/B of 5d1382f36edf9c126fa17e1f99c7f3fe into e998814dd4f84b939003d89c5c1ed246(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:25:24,199 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d1382f36edf9c126fa17e1f99c7f3fe: 2024-11-23T14:25:24,199 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., storeName=5d1382f36edf9c126fa17e1f99c7f3fe/B, priority=13, startTime=1732371924143; duration=0sec 2024-11-23T14:25:24,202 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:25:24,202 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d1382f36edf9c126fa17e1f99c7f3fe:B 2024-11-23T14:25:24,202 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:25:24,203 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:25:24,203 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1541): 5d1382f36edf9c126fa17e1f99c7f3fe/C is initiating minor compaction (all files) 2024-11-23T14:25:24,203 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d1382f36edf9c126fa17e1f99c7f3fe/C in TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:24,203 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/e9870db655984b7ba06bb5c72d4e08a8, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/d22071479c8d4828b17c68941c6cf521, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/99db463312e841d091d23483579fd35e] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp, totalSize=36.9 K 2024-11-23T14:25:24,204 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting e9870db655984b7ba06bb5c72d4e08a8, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1732371921128 2024-11-23T14:25:24,204 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting d22071479c8d4828b17c68941c6cf521, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1732371921823 2024-11-23T14:25:24,204 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 99db463312e841d091d23483579fd35e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1732371922476 2024-11-23T14:25:24,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/731630046f7c44a099a0aff9f3ed425d is 50, key is test_row_0/A:col10/1732371923189/Put/seqid=0 2024-11-23T14:25:24,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-23T14:25:24,250 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d1382f36edf9c126fa17e1f99c7f3fe#C#compaction#90 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:25:24,251 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/cdb1814505a94b87882fc80cb8482317 is 50, key is test_row_0/C:col10/1732371923144/Put/seqid=0 2024-11-23T14:25:24,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741926_1102 (size=12301) 2024-11-23T14:25:24,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741927_1103 (size=13289) 2024-11-23T14:25:24,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on 5d1382f36edf9c126fa17e1f99c7f3fe 2024-11-23T14:25:24,333 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. as already flushing 2024-11-23T14:25:24,357 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:24,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34082 deadline: 1732371984350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:24,358 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:24,358 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:24,358 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:24,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:24,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34062 deadline: 1732371984353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:24,362 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:24,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1732371984353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:24,362 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:24,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34106 deadline: 1732371984353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:24,362 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:24,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34034 deadline: 1732371984361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:24,363 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:24,363 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:24,363 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:24,364 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:24,364 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:24,364 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:24,364 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:24,364 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:24,364 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:24,364 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:24,364 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:24,364 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:24,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:24,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34062 deadline: 1732371984470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:24,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:24,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34106 deadline: 1732371984470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:24,471 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:24,471 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:24,471 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:24,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:24,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1732371984470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:24,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:24,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:24,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:24,472 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:24,472 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:24,472 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:24,605 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/d7fb76d3b4264c7998dbc2566cf97835 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/d7fb76d3b4264c7998dbc2566cf97835 2024-11-23T14:25:24,617 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5d1382f36edf9c126fa17e1f99c7f3fe/A of 5d1382f36edf9c126fa17e1f99c7f3fe into d7fb76d3b4264c7998dbc2566cf97835(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:25:24,617 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d1382f36edf9c126fa17e1f99c7f3fe: 2024-11-23T14:25:24,617 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., storeName=5d1382f36edf9c126fa17e1f99c7f3fe/A, priority=13, startTime=1732371924142; duration=0sec 2024-11-23T14:25:24,617 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:25:24,617 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d1382f36edf9c126fa17e1f99c7f3fe:A 2024-11-23T14:25:24,680 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/731630046f7c44a099a0aff9f3ed425d 2024-11-23T14:25:24,683 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:24,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34106 deadline: 1732371984681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:24,683 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:24,684 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:24,684 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:24,684 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:24,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1732371984681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:24,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:24,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:24,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:24,685 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:24,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34062 deadline: 1732371984681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:24,686 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:24,686 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:24,686 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:24,696 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/cdb1814505a94b87882fc80cb8482317 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/cdb1814505a94b87882fc80cb8482317 2024-11-23T14:25:24,704 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5d1382f36edf9c126fa17e1f99c7f3fe/C of 5d1382f36edf9c126fa17e1f99c7f3fe into cdb1814505a94b87882fc80cb8482317(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:25:24,704 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d1382f36edf9c126fa17e1f99c7f3fe: 2024-11-23T14:25:24,704 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., storeName=5d1382f36edf9c126fa17e1f99c7f3fe/C, priority=13, startTime=1732371924143; duration=0sec 2024-11-23T14:25:24,704 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:25:24,704 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d1382f36edf9c126fa17e1f99c7f3fe:C 2024-11-23T14:25:24,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/86ecbce2eb984e4b965163544209d960 is 50, key is test_row_0/B:col10/1732371923189/Put/seqid=0 2024-11-23T14:25:24,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741928_1104 (size=12301) 2024-11-23T14:25:24,753 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/86ecbce2eb984e4b965163544209d960 2024-11-23T14:25:24,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/f1a5ac9f8c1941eaa251fea32d59e1d4 is 50, key is test_row_0/C:col10/1732371923189/Put/seqid=0 2024-11-23T14:25:24,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741929_1105 (size=12301) 2024-11-23T14:25:24,814 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/f1a5ac9f8c1941eaa251fea32d59e1d4 2024-11-23T14:25:24,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/731630046f7c44a099a0aff9f3ed425d as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/731630046f7c44a099a0aff9f3ed425d 2024-11-23T14:25:24,832 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/731630046f7c44a099a0aff9f3ed425d, entries=150, sequenceid=433, filesize=12.0 K 2024-11-23T14:25:24,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/86ecbce2eb984e4b965163544209d960 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/86ecbce2eb984e4b965163544209d960 2024-11-23T14:25:24,847 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/86ecbce2eb984e4b965163544209d960, entries=150, sequenceid=433, filesize=12.0 K 2024-11-23T14:25:24,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/f1a5ac9f8c1941eaa251fea32d59e1d4 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/f1a5ac9f8c1941eaa251fea32d59e1d4 2024-11-23T14:25:24,856 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/f1a5ac9f8c1941eaa251fea32d59e1d4, entries=150, sequenceid=433, filesize=12.0 K 2024-11-23T14:25:24,857 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 5d1382f36edf9c126fa17e1f99c7f3fe in 669ms, sequenceid=433, compaction requested=false 2024-11-23T14:25:24,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for 5d1382f36edf9c126fa17e1f99c7f3fe: 2024-11-23T14:25:24,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:24,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-23T14:25:24,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-23T14:25:24,862 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-23T14:25:24,862 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7720 sec 2024-11-23T14:25:24,866 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 1.7810 sec 2024-11-23T14:25:24,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on 5d1382f36edf9c126fa17e1f99c7f3fe 2024-11-23T14:25:24,994 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T14:25:24,994 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5d1382f36edf9c126fa17e1f99c7f3fe 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-23T14:25:25,000 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/c0e63ebaf2b1431d98d8b5b471b9fef2 is 50, key is test_row_0/A:col10/1732371924350/Put/seqid=0 2024-11-23T14:25:25,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741930_1106 (size=14741) 2024-11-23T14:25:25,051 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:25,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1732371985050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:25,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:25,051 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:25,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:25,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:25,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34062 deadline: 1732371985050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:25,052 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:25,052 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:25,052 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:25,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:25,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34106 deadline: 1732371985051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:25,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:25,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:25,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:25,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:25,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1732371985160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:25,161 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:25,161 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:25,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:25,161 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:25,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34062 deadline: 1732371985160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:25,161 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:25,161 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:25,162 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:25,162 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:25,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34106 deadline: 1732371985160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:25,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:25,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:25,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:25,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-23T14:25:25,230 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T14:25:25,231 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-23T14:25:25,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-11-23T14:25:25,233 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T14:25:25,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-23T14:25:25,235 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T14:25:25,235 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T14:25:25,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-23T14:25:25,373 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:25,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34082 deadline: 1732371985370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:25,374 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to c6f7b1897123,35385,1732371902767 for 'test_row_1' in 5d1382f36edf9c126fa17e1f99c7f3fe of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-23T14:25:25,374 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:25,375 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:25,375 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:25,376 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:25,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1732371985371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:25,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:25,377 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:25,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:25,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:25,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34062 deadline: 1732371985371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:25,377 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:25,377 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:25,378 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:25,377 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:25,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34034 deadline: 1732371985371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:25,378 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:25,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34106 deadline: 1732371985371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:25,378 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:25,378 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:25,378 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:25,378 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to c6f7b1897123,35385,1732371902767 for 'test_row_2' in 5d1382f36edf9c126fa17e1f99c7f3fe of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-23T14:25:25,379 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:25,379 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:25,379 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:25,393 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-23T14:25:25,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:25,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. as already flushing 2024-11-23T14:25:25,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:25,394 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:25,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:25,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:25,406 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=453 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/c0e63ebaf2b1431d98d8b5b471b9fef2 2024-11-23T14:25:25,434 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/1f1a7e673ea84275987ecbd1c6683b00 is 50, key is test_row_0/B:col10/1732371924350/Put/seqid=0 2024-11-23T14:25:25,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741931_1107 (size=12301) 2024-11-23T14:25:25,447 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=453 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/1f1a7e673ea84275987ecbd1c6683b00 2024-11-23T14:25:25,462 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/62a23aa4fb5f4a82866df520d2709cde is 50, key is test_row_0/C:col10/1732371924350/Put/seqid=0 2024-11-23T14:25:25,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741932_1108 (size=12301) 2024-11-23T14:25:25,547 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-23T14:25:25,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:25,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. as already flushing 2024-11-23T14:25:25,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:25,548 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:25,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:25,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:25,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-23T14:25:25,693 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:25,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1732371985690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:25,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:25,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:25,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:25,695 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:25,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34106 deadline: 1732371985691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:25,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:25,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:25,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:25,696 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:25,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34062 deadline: 1732371985693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:25,697 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:25,697 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:25,697 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:25,703 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-23T14:25:25,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:25,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. as already flushing 2024-11-23T14:25:25,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:25,703 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:25,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:25,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:25,857 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-23T14:25:25,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:25,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. as already flushing 2024-11-23T14:25:25,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:25,857 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:25,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:25,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:25,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-23T14:25:25,888 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=453 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/62a23aa4fb5f4a82866df520d2709cde 2024-11-23T14:25:25,897 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/c0e63ebaf2b1431d98d8b5b471b9fef2 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/c0e63ebaf2b1431d98d8b5b471b9fef2 2024-11-23T14:25:25,906 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/c0e63ebaf2b1431d98d8b5b471b9fef2, entries=200, sequenceid=453, filesize=14.4 K 2024-11-23T14:25:25,908 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/1f1a7e673ea84275987ecbd1c6683b00 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/1f1a7e673ea84275987ecbd1c6683b00 2024-11-23T14:25:25,914 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/1f1a7e673ea84275987ecbd1c6683b00, entries=150, sequenceid=453, filesize=12.0 K 2024-11-23T14:25:25,915 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/62a23aa4fb5f4a82866df520d2709cde as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/62a23aa4fb5f4a82866df520d2709cde 2024-11-23T14:25:25,925 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/62a23aa4fb5f4a82866df520d2709cde, entries=150, sequenceid=453, filesize=12.0 K 2024-11-23T14:25:25,926 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=114.05 KB/116790 for 5d1382f36edf9c126fa17e1f99c7f3fe in 932ms, sequenceid=453, compaction requested=true 2024-11-23T14:25:25,926 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5d1382f36edf9c126fa17e1f99c7f3fe: 2024-11-23T14:25:25,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d1382f36edf9c126fa17e1f99c7f3fe:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T14:25:25,926 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:25:25,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:25:25,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d1382f36edf9c126fa17e1f99c7f3fe:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T14:25:25,926 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:25:25,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:25:25,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d1382f36edf9c126fa17e1f99c7f3fe:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T14:25:25,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:25:25,928 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:25:25,928 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1541): 5d1382f36edf9c126fa17e1f99c7f3fe/B is initiating minor compaction (all files) 2024-11-23T14:25:25,928 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d1382f36edf9c126fa17e1f99c7f3fe/B in TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:25,928 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/e998814dd4f84b939003d89c5c1ed246, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/86ecbce2eb984e4b965163544209d960, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/1f1a7e673ea84275987ecbd1c6683b00] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp, totalSize=37.0 K 2024-11-23T14:25:25,930 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40331 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:25:25,930 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting e998814dd4f84b939003d89c5c1ed246, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1732371922476 2024-11-23T14:25:25,931 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1541): 5d1382f36edf9c126fa17e1f99c7f3fe/A is initiating minor compaction (all files) 2024-11-23T14:25:25,931 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d1382f36edf9c126fa17e1f99c7f3fe/A in TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:25,931 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/d7fb76d3b4264c7998dbc2566cf97835, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/731630046f7c44a099a0aff9f3ed425d, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/c0e63ebaf2b1431d98d8b5b471b9fef2] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp, totalSize=39.4 K 2024-11-23T14:25:25,931 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 86ecbce2eb984e4b965163544209d960, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1732371923183 2024-11-23T14:25:25,931 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting d7fb76d3b4264c7998dbc2566cf97835, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1732371922476 2024-11-23T14:25:25,931 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f1a7e673ea84275987ecbd1c6683b00, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=453, earliestPutTs=1732371924350 2024-11-23T14:25:25,932 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 731630046f7c44a099a0aff9f3ed425d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1732371923183 2024-11-23T14:25:25,932 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting c0e63ebaf2b1431d98d8b5b471b9fef2, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=453, earliestPutTs=1732371924350 2024-11-23T14:25:25,944 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d1382f36edf9c126fa17e1f99c7f3fe#B#compaction#97 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:25:25,945 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d1382f36edf9c126fa17e1f99c7f3fe#A#compaction#96 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:25:25,945 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/dfc43dcd11c44bdd8728a1127f72b585 is 50, key is test_row_0/B:col10/1732371924350/Put/seqid=0 2024-11-23T14:25:25,945 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/3cf04e56c7a849e18ef8175ff57427b3 is 50, key is test_row_0/A:col10/1732371924350/Put/seqid=0 2024-11-23T14:25:25,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741933_1109 (size=13391) 2024-11-23T14:25:25,971 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/dfc43dcd11c44bdd8728a1127f72b585 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/dfc43dcd11c44bdd8728a1127f72b585 2024-11-23T14:25:25,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741934_1110 (size=13391) 2024-11-23T14:25:25,986 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/3cf04e56c7a849e18ef8175ff57427b3 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/3cf04e56c7a849e18ef8175ff57427b3 2024-11-23T14:25:25,986 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5d1382f36edf9c126fa17e1f99c7f3fe/B of 5d1382f36edf9c126fa17e1f99c7f3fe into dfc43dcd11c44bdd8728a1127f72b585(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:25:25,986 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d1382f36edf9c126fa17e1f99c7f3fe: 2024-11-23T14:25:25,986 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., storeName=5d1382f36edf9c126fa17e1f99c7f3fe/B, priority=13, startTime=1732371925926; duration=0sec 2024-11-23T14:25:25,987 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:25:25,987 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d1382f36edf9c126fa17e1f99c7f3fe:B 2024-11-23T14:25:25,987 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:25:25,989 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:25:25,989 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1541): 5d1382f36edf9c126fa17e1f99c7f3fe/C is initiating minor compaction (all files) 2024-11-23T14:25:25,989 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d1382f36edf9c126fa17e1f99c7f3fe/C in TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:25,989 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/cdb1814505a94b87882fc80cb8482317, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/f1a5ac9f8c1941eaa251fea32d59e1d4, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/62a23aa4fb5f4a82866df520d2709cde] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp, totalSize=37.0 K 2024-11-23T14:25:25,990 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting cdb1814505a94b87882fc80cb8482317, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1732371922476 2024-11-23T14:25:25,991 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting f1a5ac9f8c1941eaa251fea32d59e1d4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1732371923183 2024-11-23T14:25:25,992 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 62a23aa4fb5f4a82866df520d2709cde, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=453, earliestPutTs=1732371924350 2024-11-23T14:25:25,995 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5d1382f36edf9c126fa17e1f99c7f3fe/A of 5d1382f36edf9c126fa17e1f99c7f3fe into 3cf04e56c7a849e18ef8175ff57427b3(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:25:25,995 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d1382f36edf9c126fa17e1f99c7f3fe: 2024-11-23T14:25:25,996 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., storeName=5d1382f36edf9c126fa17e1f99c7f3fe/A, priority=13, startTime=1732371925926; duration=0sec 2024-11-23T14:25:25,998 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:25:25,998 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d1382f36edf9c126fa17e1f99c7f3fe:A 2024-11-23T14:25:26,010 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-23T14:25:26,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:26,011 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing 5d1382f36edf9c126fa17e1f99c7f3fe 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-23T14:25:26,023 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d1382f36edf9c126fa17e1f99c7f3fe#C#compaction#98 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:25:26,024 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/1414e1a990c0450a89e890cccf2e6678 is 50, key is test_row_0/C:col10/1732371924350/Put/seqid=0 2024-11-23T14:25:26,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/145a0fb715cf4e7d9a92964582b7398e is 50, key is test_row_0/A:col10/1732371925015/Put/seqid=0 2024-11-23T14:25:26,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741936_1112 (size=12301) 2024-11-23T14:25:26,048 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=475 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/145a0fb715cf4e7d9a92964582b7398e 2024-11-23T14:25:26,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741935_1111 (size=13391) 2024-11-23T14:25:26,069 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/1414e1a990c0450a89e890cccf2e6678 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/1414e1a990c0450a89e890cccf2e6678 2024-11-23T14:25:26,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/e7bc5542481b4b9eaf0ac7a7c2459711 is 50, key is test_row_0/B:col10/1732371925015/Put/seqid=0 2024-11-23T14:25:26,095 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5d1382f36edf9c126fa17e1f99c7f3fe/C of 5d1382f36edf9c126fa17e1f99c7f3fe into 1414e1a990c0450a89e890cccf2e6678(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:25:26,095 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d1382f36edf9c126fa17e1f99c7f3fe: 2024-11-23T14:25:26,095 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., storeName=5d1382f36edf9c126fa17e1f99c7f3fe/C, priority=13, startTime=1732371925927; duration=0sec 2024-11-23T14:25:26,095 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:25:26,095 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d1382f36edf9c126fa17e1f99c7f3fe:C 2024-11-23T14:25:26,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741937_1113 (size=12301) 2024-11-23T14:25:26,110 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=475 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/e7bc5542481b4b9eaf0ac7a7c2459711 2024-11-23T14:25:26,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/acc24f8aa9054f0089af4c19ca9b8ed9 is 50, key is test_row_0/C:col10/1732371925015/Put/seqid=0 2024-11-23T14:25:26,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741938_1114 (size=12301) 2024-11-23T14:25:26,203 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. as already flushing 2024-11-23T14:25:26,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on 5d1382f36edf9c126fa17e1f99c7f3fe 2024-11-23T14:25:26,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:26,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34106 deadline: 1732371986282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:26,291 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:26,291 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:26,291 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:26,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:26,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1732371986290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:26,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:26,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:26,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:26,300 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:26,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34062 deadline: 1732371986293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:26,300 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:26,300 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:26,300 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:26,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-23T14:25:26,404 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:26,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34106 deadline: 1732371986400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:26,405 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:26,405 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:26,405 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:26,405 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:26,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1732371986400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:26,405 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:26,405 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:26,405 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:26,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:26,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34062 deadline: 1732371986410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:26,413 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:26,413 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:26,413 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:26,548 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=475 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/acc24f8aa9054f0089af4c19ca9b8ed9 2024-11-23T14:25:26,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/145a0fb715cf4e7d9a92964582b7398e as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/145a0fb715cf4e7d9a92964582b7398e 2024-11-23T14:25:26,569 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/145a0fb715cf4e7d9a92964582b7398e, entries=150, sequenceid=475, filesize=12.0 K 2024-11-23T14:25:26,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/e7bc5542481b4b9eaf0ac7a7c2459711 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/e7bc5542481b4b9eaf0ac7a7c2459711 2024-11-23T14:25:26,578 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/e7bc5542481b4b9eaf0ac7a7c2459711, entries=150, sequenceid=475, filesize=12.0 K 2024-11-23T14:25:26,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/acc24f8aa9054f0089af4c19ca9b8ed9 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/acc24f8aa9054f0089af4c19ca9b8ed9 2024-11-23T14:25:26,603 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/acc24f8aa9054f0089af4c19ca9b8ed9, entries=150, sequenceid=475, filesize=12.0 K 2024-11-23T14:25:26,604 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=93.93 KB/96180 for 5d1382f36edf9c126fa17e1f99c7f3fe in 594ms, sequenceid=475, compaction requested=false 2024-11-23T14:25:26,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for 5d1382f36edf9c126fa17e1f99c7f3fe: 2024-11-23T14:25:26,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:26,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-11-23T14:25:26,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-11-23T14:25:26,607 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-23T14:25:26,608 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3710 sec 2024-11-23T14:25:26,611 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 1.3780 sec 2024-11-23T14:25:26,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on 5d1382f36edf9c126fa17e1f99c7f3fe 2024-11-23T14:25:26,618 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T14:25:26,618 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5d1382f36edf9c126fa17e1f99c7f3fe 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-23T14:25:26,630 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/0b877fae18ed4b9faa30a2ae34bd1826 is 50, key is test_row_0/A:col10/1732371926272/Put/seqid=0 2024-11-23T14:25:26,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741939_1115 (size=14741) 2024-11-23T14:25:26,648 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=495 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/0b877fae18ed4b9faa30a2ae34bd1826 2024-11-23T14:25:26,662 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/5df6151be6ce44ab808136c8ef2b1259 is 50, key is test_row_0/B:col10/1732371926272/Put/seqid=0 2024-11-23T14:25:26,668 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:26,668 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:26,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34106 deadline: 1732371986658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:26,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34062 deadline: 1732371986659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:26,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:26,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:26,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:26,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:26,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1732371986665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:26,669 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:26,669 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:26,669 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:26,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:26,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:26,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:26,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741940_1116 (size=12301) 2024-11-23T14:25:26,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:26,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34106 deadline: 1732371986780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:26,784 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:26,784 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:26,784 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:26,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:26,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1732371986780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:26,787 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:26,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34062 deadline: 1732371986780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:26,787 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:26,787 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:26,787 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:26,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:26,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:26,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:26,992 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:26,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34106 deadline: 1732371986990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:26,993 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:26,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:26,993 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:26,993 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:26,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34062 deadline: 1732371986991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:26,993 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:26,993 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:26,993 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:27,003 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:27,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1732371987000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:27,003 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:27,003 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:27,004 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:27,098 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=495 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/5df6151be6ce44ab808136c8ef2b1259 2024-11-23T14:25:27,107 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/466af783f01544229a38f9e94eba363a is 50, key is test_row_0/C:col10/1732371926272/Put/seqid=0 2024-11-23T14:25:27,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741941_1117 (size=12301) 2024-11-23T14:25:27,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:27,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34062 deadline: 1732371987300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:27,302 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:27,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34106 deadline: 1732371987300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:27,302 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:27,302 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:27,302 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:27,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:27,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:27,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:27,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:27,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1732371987310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:27,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:27,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:27,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:27,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-23T14:25:27,381 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T14:25:27,381 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-23T14:25:27,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees 2024-11-23T14:25:27,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-23T14:25:27,389 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T14:25:27,392 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T14:25:27,392 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T14:25:27,401 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:27,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34082 deadline: 1732371987400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:27,402 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:27,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34034 deadline: 1732371987401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:27,402 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to c6f7b1897123,35385,1732371902767 for 'test_row_1' in 5d1382f36edf9c126fa17e1f99c7f3fe of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-23T14:25:27,402 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to c6f7b1897123,35385,1732371902767 for 'test_row_2' in 5d1382f36edf9c126fa17e1f99c7f3fe of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-23T14:25:27,403 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:27,403 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:27,403 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:27,403 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:27,403 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:27,403 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:27,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-23T14:25:27,515 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=495 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/466af783f01544229a38f9e94eba363a 2024-11-23T14:25:27,522 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/0b877fae18ed4b9faa30a2ae34bd1826 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/0b877fae18ed4b9faa30a2ae34bd1826 2024-11-23T14:25:27,529 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/0b877fae18ed4b9faa30a2ae34bd1826, entries=200, sequenceid=495, filesize=14.4 K 2024-11-23T14:25:27,531 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/5df6151be6ce44ab808136c8ef2b1259 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/5df6151be6ce44ab808136c8ef2b1259 2024-11-23T14:25:27,541 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/5df6151be6ce44ab808136c8ef2b1259, entries=150, sequenceid=495, filesize=12.0 K 2024-11-23T14:25:27,543 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/466af783f01544229a38f9e94eba363a as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/466af783f01544229a38f9e94eba363a 2024-11-23T14:25:27,548 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-23T14:25:27,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:27,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. as already flushing 2024-11-23T14:25:27,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:27,549 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:27,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:27,549 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/466af783f01544229a38f9e94eba363a, entries=150, sequenceid=495, filesize=12.0 K 2024-11-23T14:25:27,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:27,550 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 5d1382f36edf9c126fa17e1f99c7f3fe in 932ms, sequenceid=495, compaction requested=true 2024-11-23T14:25:27,551 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5d1382f36edf9c126fa17e1f99c7f3fe: 2024-11-23T14:25:27,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d1382f36edf9c126fa17e1f99c7f3fe:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T14:25:27,551 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:25:27,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:25:27,551 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:25:27,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d1382f36edf9c126fa17e1f99c7f3fe:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T14:25:27,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:25:27,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d1382f36edf9c126fa17e1f99c7f3fe:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T14:25:27,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:25:27,553 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37993 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:25:27,553 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1541): 5d1382f36edf9c126fa17e1f99c7f3fe/B is initiating minor compaction (all files) 2024-11-23T14:25:27,553 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d1382f36edf9c126fa17e1f99c7f3fe/B in TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:27,554 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/dfc43dcd11c44bdd8728a1127f72b585, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/e7bc5542481b4b9eaf0ac7a7c2459711, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/5df6151be6ce44ab808136c8ef2b1259] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp, totalSize=37.1 K 2024-11-23T14:25:27,554 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40433 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:25:27,554 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1541): 5d1382f36edf9c126fa17e1f99c7f3fe/A is initiating minor compaction (all files) 2024-11-23T14:25:27,554 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d1382f36edf9c126fa17e1f99c7f3fe/A in TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:27,554 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting dfc43dcd11c44bdd8728a1127f72b585, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=453, earliestPutTs=1732371924350 2024-11-23T14:25:27,554 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/3cf04e56c7a849e18ef8175ff57427b3, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/145a0fb715cf4e7d9a92964582b7398e, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/0b877fae18ed4b9faa30a2ae34bd1826] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp, totalSize=39.5 K 2024-11-23T14:25:27,555 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting e7bc5542481b4b9eaf0ac7a7c2459711, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=475, earliestPutTs=1732371925011 2024-11-23T14:25:27,555 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3cf04e56c7a849e18ef8175ff57427b3, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=453, earliestPutTs=1732371924350 2024-11-23T14:25:27,555 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 5df6151be6ce44ab808136c8ef2b1259, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=495, earliestPutTs=1732371926272 2024-11-23T14:25:27,555 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 145a0fb715cf4e7d9a92964582b7398e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=475, earliestPutTs=1732371925011 2024-11-23T14:25:27,556 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b877fae18ed4b9faa30a2ae34bd1826, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=495, earliestPutTs=1732371926272 2024-11-23T14:25:27,571 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d1382f36edf9c126fa17e1f99c7f3fe#B#compaction#106 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:25:27,571 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d1382f36edf9c126fa17e1f99c7f3fe#A#compaction#105 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:25:27,571 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/3db686dd3cc74988802cb13fef50854f is 50, key is test_row_0/B:col10/1732371926272/Put/seqid=0 2024-11-23T14:25:27,571 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/0e9ed7cdc8da43b88ff151bea1ccd7a9 is 50, key is test_row_0/A:col10/1732371926272/Put/seqid=0 2024-11-23T14:25:27,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741943_1119 (size=13493) 2024-11-23T14:25:27,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741942_1118 (size=13493) 2024-11-23T14:25:27,614 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/3db686dd3cc74988802cb13fef50854f as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/3db686dd3cc74988802cb13fef50854f 2024-11-23T14:25:27,623 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/0e9ed7cdc8da43b88ff151bea1ccd7a9 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/0e9ed7cdc8da43b88ff151bea1ccd7a9 2024-11-23T14:25:27,633 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5d1382f36edf9c126fa17e1f99c7f3fe/B of 5d1382f36edf9c126fa17e1f99c7f3fe into 3db686dd3cc74988802cb13fef50854f(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:25:27,633 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d1382f36edf9c126fa17e1f99c7f3fe: 2024-11-23T14:25:27,633 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., storeName=5d1382f36edf9c126fa17e1f99c7f3fe/B, priority=13, startTime=1732371927551; duration=0sec 2024-11-23T14:25:27,633 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:25:27,633 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d1382f36edf9c126fa17e1f99c7f3fe:B 2024-11-23T14:25:27,633 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:25:27,634 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5d1382f36edf9c126fa17e1f99c7f3fe/A of 5d1382f36edf9c126fa17e1f99c7f3fe into 0e9ed7cdc8da43b88ff151bea1ccd7a9(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:25:27,634 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d1382f36edf9c126fa17e1f99c7f3fe: 2024-11-23T14:25:27,634 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., storeName=5d1382f36edf9c126fa17e1f99c7f3fe/A, priority=13, startTime=1732371927551; duration=0sec 2024-11-23T14:25:27,635 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:25:27,635 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d1382f36edf9c126fa17e1f99c7f3fe:A 2024-11-23T14:25:27,636 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37993 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:25:27,636 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1541): 5d1382f36edf9c126fa17e1f99c7f3fe/C is initiating minor compaction (all files) 2024-11-23T14:25:27,637 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d1382f36edf9c126fa17e1f99c7f3fe/C in TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:27,637 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/1414e1a990c0450a89e890cccf2e6678, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/acc24f8aa9054f0089af4c19ca9b8ed9, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/466af783f01544229a38f9e94eba363a] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp, totalSize=37.1 K 2024-11-23T14:25:27,637 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 1414e1a990c0450a89e890cccf2e6678, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=453, earliestPutTs=1732371924350 2024-11-23T14:25:27,638 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting acc24f8aa9054f0089af4c19ca9b8ed9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=475, earliestPutTs=1732371925011 2024-11-23T14:25:27,638 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 466af783f01544229a38f9e94eba363a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=495, earliestPutTs=1732371926272 2024-11-23T14:25:27,648 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d1382f36edf9c126fa17e1f99c7f3fe#C#compaction#107 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:25:27,648 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/62cb55ba63c248ad86fb2f6a21abc48d is 50, key is test_row_0/C:col10/1732371926272/Put/seqid=0 2024-11-23T14:25:27,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741944_1120 (size=13493) 2024-11-23T14:25:27,702 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-23T14:25:27,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:27,702 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2902): Flushing 5d1382f36edf9c126fa17e1f99c7f3fe 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-23T14:25:27,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/41c6b432b77c4512bb2f99030ec78167 is 50, key is test_row_0/A:col10/1732371926643/Put/seqid=0 2024-11-23T14:25:27,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-23T14:25:27,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741945_1121 (size=12301) 2024-11-23T14:25:27,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on 5d1382f36edf9c126fa17e1f99c7f3fe 2024-11-23T14:25:27,812 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. as already flushing 2024-11-23T14:25:27,845 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:27,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34106 deadline: 1732371987841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:27,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:27,846 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:27,846 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:27,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34062 deadline: 1732371987842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:27,846 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:27,846 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:27,847 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:27,847 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:27,847 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:27,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1732371987844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:27,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:27,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:27,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:27,953 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:27,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34062 deadline: 1732371987950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:27,954 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:27,954 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:27,954 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:27,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:27,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34106 deadline: 1732371987950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:27,955 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:27,955 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:27,955 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:27,955 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:27,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1732371987950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:27,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:27,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:27,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:28,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-23T14:25:28,084 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/62cb55ba63c248ad86fb2f6a21abc48d as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/62cb55ba63c248ad86fb2f6a21abc48d 2024-11-23T14:25:28,091 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5d1382f36edf9c126fa17e1f99c7f3fe/C of 5d1382f36edf9c126fa17e1f99c7f3fe into 62cb55ba63c248ad86fb2f6a21abc48d(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:25:28,091 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d1382f36edf9c126fa17e1f99c7f3fe: 2024-11-23T14:25:28,091 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., storeName=5d1382f36edf9c126fa17e1f99c7f3fe/C, priority=13, startTime=1732371927551; duration=0sec 2024-11-23T14:25:28,092 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:25:28,092 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d1382f36edf9c126fa17e1f99c7f3fe:C 2024-11-23T14:25:28,123 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=514 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/41c6b432b77c4512bb2f99030ec78167 2024-11-23T14:25:28,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/0be7e18aaba84fb3bd9c626668e3eed9 is 50, key is test_row_0/B:col10/1732371926643/Put/seqid=0 2024-11-23T14:25:28,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741946_1122 (size=12301) 2024-11-23T14:25:28,137 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=514 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/0be7e18aaba84fb3bd9c626668e3eed9 2024-11-23T14:25:28,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/f95e6348c8da4e3f8d53e862e0b8148b is 50, key is test_row_0/C:col10/1732371926643/Put/seqid=0 2024-11-23T14:25:28,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741947_1123 (size=12301) 2024-11-23T14:25:28,161 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=514 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/f95e6348c8da4e3f8d53e862e0b8148b 2024-11-23T14:25:28,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:28,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34062 deadline: 1732371988160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:28,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:28,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34106 deadline: 1732371988160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:28,164 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:28,164 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:28,164 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:28,164 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:28,164 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:28,164 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:28,165 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:28,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1732371988160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:28,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:28,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:28,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:28,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/41c6b432b77c4512bb2f99030ec78167 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/41c6b432b77c4512bb2f99030ec78167 2024-11-23T14:25:28,188 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/41c6b432b77c4512bb2f99030ec78167, entries=150, sequenceid=514, filesize=12.0 K 2024-11-23T14:25:28,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/0be7e18aaba84fb3bd9c626668e3eed9 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/0be7e18aaba84fb3bd9c626668e3eed9 2024-11-23T14:25:28,201 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/0be7e18aaba84fb3bd9c626668e3eed9, entries=150, sequenceid=514, filesize=12.0 K 2024-11-23T14:25:28,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/f95e6348c8da4e3f8d53e862e0b8148b as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/f95e6348c8da4e3f8d53e862e0b8148b 2024-11-23T14:25:28,214 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/f95e6348c8da4e3f8d53e862e0b8148b, entries=150, sequenceid=514, filesize=12.0 K 2024-11-23T14:25:28,216 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 5d1382f36edf9c126fa17e1f99c7f3fe in 513ms, sequenceid=514, compaction requested=false 2024-11-23T14:25:28,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2603): Flush status journal for 5d1382f36edf9c126fa17e1f99c7f3fe: 2024-11-23T14:25:28,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:28,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=28 2024-11-23T14:25:28,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4169): Remote procedure done, pid=28 2024-11-23T14:25:28,222 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-23T14:25:28,222 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 826 msec 2024-11-23T14:25:28,229 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees in 843 msec 2024-11-23T14:25:28,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on 5d1382f36edf9c126fa17e1f99c7f3fe 2024-11-23T14:25:28,471 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T14:25:28,471 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5d1382f36edf9c126fa17e1f99c7f3fe 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-23T14:25:28,477 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/58f2638db90c434cb0dcd300494d37d3 is 50, key is test_row_0/A:col10/1732371928470/Put/seqid=0 2024-11-23T14:25:28,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741948_1124 (size=17181) 2024-11-23T14:25:28,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:28,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34106 deadline: 1732371988498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:28,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:28,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:28,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34062 deadline: 1732371988498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:28,503 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:28,503 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:28,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:28,503 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:28,503 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:28,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1732371988499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:28,503 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:28,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:28,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:28,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:28,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-23T14:25:28,531 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T14:25:28,531 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-23T14:25:28,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=29, table=TestAcidGuarantees 2024-11-23T14:25:28,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-23T14:25:28,534 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=29, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=29, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T14:25:28,534 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=29, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=29, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T14:25:28,535 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T14:25:28,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:28,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34106 deadline: 1732371988610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:28,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:28,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:28,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:28,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34062 deadline: 1732371988610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:28,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:28,612 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:28,612 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:28,612 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:28,612 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:28,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1732371988610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:28,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:28,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:28,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:28,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-23T14:25:28,687 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-11-23T14:25:28,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:28,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. as already flushing 2024-11-23T14:25:28,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:28,687 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] handler.RSProcedureHandler(58): pid=30 java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:28,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=30 java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:28,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=30 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:28,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:28,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34062 deadline: 1732371988820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:28,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:28,822 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:28,822 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:28,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34106 deadline: 1732371988820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:28,822 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:28,822 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:28,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:28,822 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:28,822 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:28,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1732371988820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:28,824 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:28,824 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:28,824 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:28,842 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-11-23T14:25:28,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:28,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. as already flushing 2024-11-23T14:25:28,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:28,842 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] handler.RSProcedureHandler(58): pid=30 java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:28,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=30 java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:28,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=30 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:28,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-23T14:25:28,891 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=536 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/58f2638db90c434cb0dcd300494d37d3 2024-11-23T14:25:28,903 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/a6ff97e205c04b70951cfb19ce41fd2c is 50, key is test_row_0/B:col10/1732371928470/Put/seqid=0 2024-11-23T14:25:28,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741949_1125 (size=12301) 2024-11-23T14:25:28,928 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=536 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/a6ff97e205c04b70951cfb19ce41fd2c 2024-11-23T14:25:28,938 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/b2157d6fd72445a88ac8a4df6bf2ee6f is 50, key is test_row_0/C:col10/1732371928470/Put/seqid=0 2024-11-23T14:25:28,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741950_1126 (size=12301) 2024-11-23T14:25:28,952 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=536 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/b2157d6fd72445a88ac8a4df6bf2ee6f 2024-11-23T14:25:28,974 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/58f2638db90c434cb0dcd300494d37d3 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/58f2638db90c434cb0dcd300494d37d3 2024-11-23T14:25:28,994 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/58f2638db90c434cb0dcd300494d37d3, entries=250, sequenceid=536, filesize=16.8 K 2024-11-23T14:25:28,995 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-11-23T14:25:28,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:28,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. as already flushing 2024-11-23T14:25:28,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:28,996 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] handler.RSProcedureHandler(58): pid=30 java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:28,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=30 java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:28,996 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/a6ff97e205c04b70951cfb19ce41fd2c as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/a6ff97e205c04b70951cfb19ce41fd2c 2024-11-23T14:25:28,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=30 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:29,008 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/a6ff97e205c04b70951cfb19ce41fd2c, entries=150, sequenceid=536, filesize=12.0 K 2024-11-23T14:25:29,009 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/b2157d6fd72445a88ac8a4df6bf2ee6f as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/b2157d6fd72445a88ac8a4df6bf2ee6f 2024-11-23T14:25:29,024 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/b2157d6fd72445a88ac8a4df6bf2ee6f, entries=150, sequenceid=536, filesize=12.0 K 2024-11-23T14:25:29,028 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=87.22 KB/89310 for 5d1382f36edf9c126fa17e1f99c7f3fe in 556ms, sequenceid=536, compaction requested=true 2024-11-23T14:25:29,028 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5d1382f36edf9c126fa17e1f99c7f3fe: 2024-11-23T14:25:29,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d1382f36edf9c126fa17e1f99c7f3fe:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T14:25:29,028 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:25:29,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:25:29,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d1382f36edf9c126fa17e1f99c7f3fe:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T14:25:29,028 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:25:29,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:25:29,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d1382f36edf9c126fa17e1f99c7f3fe:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T14:25:29,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:25:29,029 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38095 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:25:29,029 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42975 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:25:29,029 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1541): 5d1382f36edf9c126fa17e1f99c7f3fe/A is initiating minor compaction (all files) 2024-11-23T14:25:29,029 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1541): 5d1382f36edf9c126fa17e1f99c7f3fe/B is initiating minor compaction (all files) 2024-11-23T14:25:29,029 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d1382f36edf9c126fa17e1f99c7f3fe/A in TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:29,029 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d1382f36edf9c126fa17e1f99c7f3fe/B in TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:29,029 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/0e9ed7cdc8da43b88ff151bea1ccd7a9, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/41c6b432b77c4512bb2f99030ec78167, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/58f2638db90c434cb0dcd300494d37d3] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp, totalSize=42.0 K 2024-11-23T14:25:29,029 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/3db686dd3cc74988802cb13fef50854f, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/0be7e18aaba84fb3bd9c626668e3eed9, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/a6ff97e205c04b70951cfb19ce41fd2c] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp, totalSize=37.2 K 2024-11-23T14:25:29,030 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 3db686dd3cc74988802cb13fef50854f, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=495, earliestPutTs=1732371926272 2024-11-23T14:25:29,030 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e9ed7cdc8da43b88ff151bea1ccd7a9, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=495, earliestPutTs=1732371926272 2024-11-23T14:25:29,030 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 0be7e18aaba84fb3bd9c626668e3eed9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=514, earliestPutTs=1732371926643 2024-11-23T14:25:29,030 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 41c6b432b77c4512bb2f99030ec78167, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=514, earliestPutTs=1732371926643 2024-11-23T14:25:29,031 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting a6ff97e205c04b70951cfb19ce41fd2c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=536, earliestPutTs=1732371927842 2024-11-23T14:25:29,032 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 58f2638db90c434cb0dcd300494d37d3, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=536, earliestPutTs=1732371927822 2024-11-23T14:25:29,047 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d1382f36edf9c126fa17e1f99c7f3fe#B#compaction#114 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:25:29,048 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/ec61aecc634042a4af3040b0d7490cd8 is 50, key is test_row_0/B:col10/1732371928470/Put/seqid=0 2024-11-23T14:25:29,053 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d1382f36edf9c126fa17e1f99c7f3fe#A#compaction#115 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:25:29,054 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/7c9308a724f7479ab0af528a5e8cb484 is 50, key is test_row_0/A:col10/1732371928470/Put/seqid=0 2024-11-23T14:25:29,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741951_1127 (size=13595) 2024-11-23T14:25:29,069 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/ec61aecc634042a4af3040b0d7490cd8 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/ec61aecc634042a4af3040b0d7490cd8 2024-11-23T14:25:29,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741952_1128 (size=13595) 2024-11-23T14:25:29,076 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/7c9308a724f7479ab0af528a5e8cb484 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/7c9308a724f7479ab0af528a5e8cb484 2024-11-23T14:25:29,078 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5d1382f36edf9c126fa17e1f99c7f3fe/B of 5d1382f36edf9c126fa17e1f99c7f3fe into ec61aecc634042a4af3040b0d7490cd8(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:25:29,078 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d1382f36edf9c126fa17e1f99c7f3fe: 2024-11-23T14:25:29,078 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., storeName=5d1382f36edf9c126fa17e1f99c7f3fe/B, priority=13, startTime=1732371929028; duration=0sec 2024-11-23T14:25:29,078 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:25:29,078 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d1382f36edf9c126fa17e1f99c7f3fe:B 2024-11-23T14:25:29,079 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:25:29,081 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38095 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:25:29,081 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1541): 5d1382f36edf9c126fa17e1f99c7f3fe/C is initiating minor compaction (all files) 2024-11-23T14:25:29,081 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d1382f36edf9c126fa17e1f99c7f3fe/C in TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:29,081 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/62cb55ba63c248ad86fb2f6a21abc48d, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/f95e6348c8da4e3f8d53e862e0b8148b, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/b2157d6fd72445a88ac8a4df6bf2ee6f] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp, totalSize=37.2 K 2024-11-23T14:25:29,082 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 62cb55ba63c248ad86fb2f6a21abc48d, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=495, earliestPutTs=1732371926272 2024-11-23T14:25:29,082 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting f95e6348c8da4e3f8d53e862e0b8148b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=514, earliestPutTs=1732371926643 2024-11-23T14:25:29,083 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting b2157d6fd72445a88ac8a4df6bf2ee6f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=536, earliestPutTs=1732371927842 2024-11-23T14:25:29,083 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5d1382f36edf9c126fa17e1f99c7f3fe/A of 5d1382f36edf9c126fa17e1f99c7f3fe into 7c9308a724f7479ab0af528a5e8cb484(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:25:29,083 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d1382f36edf9c126fa17e1f99c7f3fe: 2024-11-23T14:25:29,083 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., storeName=5d1382f36edf9c126fa17e1f99c7f3fe/A, priority=13, startTime=1732371929028; duration=0sec 2024-11-23T14:25:29,083 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:25:29,083 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d1382f36edf9c126fa17e1f99c7f3fe:A 2024-11-23T14:25:29,091 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d1382f36edf9c126fa17e1f99c7f3fe#C#compaction#116 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:25:29,092 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/4b1ad7c0bf0b49b2b36e897a310a2ab4 is 50, key is test_row_0/C:col10/1732371928470/Put/seqid=0 2024-11-23T14:25:29,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741953_1129 (size=13595) 2024-11-23T14:25:29,105 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/4b1ad7c0bf0b49b2b36e897a310a2ab4 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/4b1ad7c0bf0b49b2b36e897a310a2ab4 2024-11-23T14:25:29,115 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5d1382f36edf9c126fa17e1f99c7f3fe/C of 5d1382f36edf9c126fa17e1f99c7f3fe into 4b1ad7c0bf0b49b2b36e897a310a2ab4(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:25:29,115 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d1382f36edf9c126fa17e1f99c7f3fe: 2024-11-23T14:25:29,115 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., storeName=5d1382f36edf9c126fa17e1f99c7f3fe/C, priority=13, startTime=1732371929028; duration=0sec 2024-11-23T14:25:29,116 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:25:29,116 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d1382f36edf9c126fa17e1f99c7f3fe:C 2024-11-23T14:25:29,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on 5d1382f36edf9c126fa17e1f99c7f3fe 2024-11-23T14:25:29,131 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T14:25:29,131 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5d1382f36edf9c126fa17e1f99c7f3fe 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-23T14:25:29,136 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/49f73e9539584f6cbff3fcc5bb888d2d is 50, key is test_row_0/A:col10/1732371928497/Put/seqid=0 2024-11-23T14:25:29,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741954_1130 (size=14741) 2024-11-23T14:25:29,142 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=556 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/49f73e9539584f6cbff3fcc5bb888d2d 2024-11-23T14:25:29,151 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-11-23T14:25:29,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:29,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. as already flushing 2024-11-23T14:25:29,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:29,152 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/37ac88b1bf3a46aaac1274c9936914c9 is 50, key is test_row_0/B:col10/1732371928497/Put/seqid=0 2024-11-23T14:25:29,152 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] handler.RSProcedureHandler(58): pid=30 java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:29,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=30 java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:29,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=30 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:29,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:29,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-23T14:25:29,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34106 deadline: 1732371989156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:29,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:29,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:29,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:29,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34062 deadline: 1732371989156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:29,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:29,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:29,161 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:29,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1732371989156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:29,161 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:29,161 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:29,161 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:29,161 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:29,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:29,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741955_1131 (size=12301) 2024-11-23T14:25:29,175 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=556 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/37ac88b1bf3a46aaac1274c9936914c9 2024-11-23T14:25:29,184 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/6e4616ba5294452b81278698593707a4 is 50, key is test_row_0/C:col10/1732371928497/Put/seqid=0 2024-11-23T14:25:29,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741956_1132 (size=12301) 2024-11-23T14:25:29,189 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=556 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/6e4616ba5294452b81278698593707a4 2024-11-23T14:25:29,194 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/49f73e9539584f6cbff3fcc5bb888d2d as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/49f73e9539584f6cbff3fcc5bb888d2d 2024-11-23T14:25:29,199 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/49f73e9539584f6cbff3fcc5bb888d2d, entries=200, sequenceid=556, filesize=14.4 K 2024-11-23T14:25:29,201 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/37ac88b1bf3a46aaac1274c9936914c9 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/37ac88b1bf3a46aaac1274c9936914c9 2024-11-23T14:25:29,206 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/37ac88b1bf3a46aaac1274c9936914c9, entries=150, sequenceid=556, filesize=12.0 K 2024-11-23T14:25:29,207 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/6e4616ba5294452b81278698593707a4 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/6e4616ba5294452b81278698593707a4 2024-11-23T14:25:29,212 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/6e4616ba5294452b81278698593707a4, entries=150, sequenceid=556, filesize=12.0 K 2024-11-23T14:25:29,213 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=114.05 KB/116790 for 5d1382f36edf9c126fa17e1f99c7f3fe in 82ms, sequenceid=556, compaction requested=false 2024-11-23T14:25:29,213 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5d1382f36edf9c126fa17e1f99c7f3fe: 2024-11-23T14:25:29,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on 5d1382f36edf9c126fa17e1f99c7f3fe 2024-11-23T14:25:29,300 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T14:25:29,301 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5d1382f36edf9c126fa17e1f99c7f3fe 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-23T14:25:29,306 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/13a63584e98343ad8fc150995c7b116f is 50, key is test_row_0/A:col10/1732371929273/Put/seqid=0 2024-11-23T14:25:29,306 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-11-23T14:25:29,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:29,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. as already flushing 2024-11-23T14:25:29,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:29,306 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] handler.RSProcedureHandler(58): pid=30 java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:29,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=30 java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:29,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=30 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:29,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741957_1133 (size=12301) 2024-11-23T14:25:29,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:29,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34106 deadline: 1732371989324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:29,327 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:29,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34062 deadline: 1732371989325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:29,327 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:29,327 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:29,327 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:29,327 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:29,327 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:29,327 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:29,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:29,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1732371989327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:29,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:29,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:29,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:29,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:29,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34106 deadline: 1732371989430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:29,431 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:29,431 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:29,431 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:29,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34062 deadline: 1732371989430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:29,431 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:29,431 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:29,431 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:29,431 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:29,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:29,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1732371989440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:29,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:29,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:29,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:29,459 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-11-23T14:25:29,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:29,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. as already flushing 2024-11-23T14:25:29,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:29,460 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] handler.RSProcedureHandler(58): pid=30 java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:29,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=30 java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:29,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=30 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:29,612 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-11-23T14:25:29,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:29,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. as already flushing 2024-11-23T14:25:29,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:29,613 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] handler.RSProcedureHandler(58): pid=30 java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:29,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=30 java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:29,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=30 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:29,640 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:29,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34106 deadline: 1732371989640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:29,640 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:29,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34062 deadline: 1732371989640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:29,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:29,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:29,641 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:29,641 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:29,641 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:29,641 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:29,654 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:29,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1732371989650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:29,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:29,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:29,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:29,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-23T14:25:29,718 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=579 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/13a63584e98343ad8fc150995c7b116f 2024-11-23T14:25:29,727 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/f37b505933bd48a1ae56d66145e0f6bf is 50, key is test_row_0/B:col10/1732371929273/Put/seqid=0 2024-11-23T14:25:29,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741958_1134 (size=12301) 2024-11-23T14:25:29,765 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-11-23T14:25:29,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:29,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. as already flushing 2024-11-23T14:25:29,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:29,766 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] handler.RSProcedureHandler(58): pid=30 java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:29,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=30 java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:29,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=30 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:29,828 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-23T14:25:29,829 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T14:25:29,831 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-23T14:25:29,831 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T14:25:29,833 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:25:29,833 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:25:29,833 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:25:29,833 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T14:25:29,834 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T14:25:29,834 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:25:29,838 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-23T14:25:29,838 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T14:25:29,838 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:25:29,838 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:25:29,838 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T14:25:29,838 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-23T14:25:29,838 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T14:25:29,838 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:25:29,838 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:25:29,838 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T14:25:29,919 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-11-23T14:25:29,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:29,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. as already flushing 2024-11-23T14:25:29,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:29,920 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] handler.RSProcedureHandler(58): pid=30 java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:29,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=30 java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:29,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=30 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:29,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:29,951 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:29,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34106 deadline: 1732371989950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:29,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34062 deadline: 1732371989950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:29,952 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:29,952 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:29,952 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:29,952 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:29,952 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:29,952 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:29,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:29,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1732371989960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:29,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:29,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:29,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:30,077 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-11-23T14:25:30,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:30,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. as already flushing 2024-11-23T14:25:30,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:30,078 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] handler.RSProcedureHandler(58): pid=30 java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:30,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=30 java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:30,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=30 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:30,134 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=579 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/f37b505933bd48a1ae56d66145e0f6bf 2024-11-23T14:25:30,149 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/cc09233146984fc4924bc233e706d593 is 50, key is test_row_0/C:col10/1732371929273/Put/seqid=0 2024-11-23T14:25:30,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741959_1135 (size=12301) 2024-11-23T14:25:30,235 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-11-23T14:25:30,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:30,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. as already flushing 2024-11-23T14:25:30,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:30,235 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] handler.RSProcedureHandler(58): pid=30 java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:30,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=30 java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:30,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=30 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:30,391 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-11-23T14:25:30,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:30,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. as already flushing 2024-11-23T14:25:30,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:30,392 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] handler.RSProcedureHandler(58): pid=30 java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:30,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=30 java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:30,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=30 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:30,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:30,462 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:30,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34062 deadline: 1732371990461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:30,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34106 deadline: 1732371990461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:30,463 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:30,463 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:30,463 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:30,463 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:30,463 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:30,463 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:30,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:30,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1732371990471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:30,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:30,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:30,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:25:30,550 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-11-23T14:25:30,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:30,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. as already flushing 2024-11-23T14:25:30,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:30,551 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] handler.RSProcedureHandler(58): pid=30 java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:30,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=30 java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:30,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=30 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:30,555 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=579 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/cc09233146984fc4924bc233e706d593 2024-11-23T14:25:30,563 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/13a63584e98343ad8fc150995c7b116f as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/13a63584e98343ad8fc150995c7b116f 2024-11-23T14:25:30,568 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/13a63584e98343ad8fc150995c7b116f, entries=150, sequenceid=579, filesize=12.0 K 2024-11-23T14:25:30,569 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/f37b505933bd48a1ae56d66145e0f6bf as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/f37b505933bd48a1ae56d66145e0f6bf 2024-11-23T14:25:30,575 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/f37b505933bd48a1ae56d66145e0f6bf, entries=150, sequenceid=579, filesize=12.0 K 2024-11-23T14:25:30,576 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/cc09233146984fc4924bc233e706d593 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/cc09233146984fc4924bc233e706d593 2024-11-23T14:25:30,582 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/cc09233146984fc4924bc233e706d593, entries=150, sequenceid=579, filesize=12.0 K 2024-11-23T14:25:30,583 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 5d1382f36edf9c126fa17e1f99c7f3fe in 1283ms, sequenceid=579, compaction requested=true 2024-11-23T14:25:30,583 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5d1382f36edf9c126fa17e1f99c7f3fe: 2024-11-23T14:25:30,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d1382f36edf9c126fa17e1f99c7f3fe:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T14:25:30,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:25:30,583 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:25:30,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d1382f36edf9c126fa17e1f99c7f3fe:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T14:25:30,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:25:30,583 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:25:30,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d1382f36edf9c126fa17e1f99c7f3fe:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T14:25:30,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:25:30,585 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38197 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:25:30,585 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40637 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:25:30,585 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1541): 5d1382f36edf9c126fa17e1f99c7f3fe/B is initiating minor compaction (all files) 2024-11-23T14:25:30,585 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1541): 5d1382f36edf9c126fa17e1f99c7f3fe/A is initiating minor compaction (all files) 2024-11-23T14:25:30,585 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d1382f36edf9c126fa17e1f99c7f3fe/B in TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:30,585 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d1382f36edf9c126fa17e1f99c7f3fe/A in TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:30,585 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/ec61aecc634042a4af3040b0d7490cd8, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/37ac88b1bf3a46aaac1274c9936914c9, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/f37b505933bd48a1ae56d66145e0f6bf] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp, totalSize=37.3 K 2024-11-23T14:25:30,585 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/7c9308a724f7479ab0af528a5e8cb484, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/49f73e9539584f6cbff3fcc5bb888d2d, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/13a63584e98343ad8fc150995c7b116f] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp, totalSize=39.7 K 2024-11-23T14:25:30,585 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting ec61aecc634042a4af3040b0d7490cd8, keycount=150, bloomtype=ROW, size=13.3 K, encoding=NONE, compression=NONE, seqNum=536, earliestPutTs=1732371927842 2024-11-23T14:25:30,585 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c9308a724f7479ab0af528a5e8cb484, keycount=150, bloomtype=ROW, size=13.3 K, encoding=NONE, compression=NONE, seqNum=536, earliestPutTs=1732371927842 2024-11-23T14:25:30,586 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 37ac88b1bf3a46aaac1274c9936914c9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=556, earliestPutTs=1732371928491 2024-11-23T14:25:30,586 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49f73e9539584f6cbff3fcc5bb888d2d, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=556, earliestPutTs=1732371928476 2024-11-23T14:25:30,586 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting f37b505933bd48a1ae56d66145e0f6bf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=579, earliestPutTs=1732371929270 2024-11-23T14:25:30,586 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 13a63584e98343ad8fc150995c7b116f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=579, earliestPutTs=1732371929270 2024-11-23T14:25:30,596 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d1382f36edf9c126fa17e1f99c7f3fe#A#compaction#124 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:25:30,597 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d1382f36edf9c126fa17e1f99c7f3fe#B#compaction#123 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:25:30,597 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/ce365a2d62284ab49efafe69e2d61370 is 50, key is test_row_0/A:col10/1732371929273/Put/seqid=0 2024-11-23T14:25:30,597 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/434b136c4a2d4b488a22f926be4e4e4d is 50, key is test_row_0/B:col10/1732371929273/Put/seqid=0 2024-11-23T14:25:30,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741960_1136 (size=13697) 2024-11-23T14:25:30,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741961_1137 (size=13697) 2024-11-23T14:25:30,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-23T14:25:30,709 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-11-23T14:25:30,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:30,710 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2902): Flushing 5d1382f36edf9c126fa17e1f99c7f3fe 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-23T14:25:30,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/028f0115340740838d71dbb5b411c1bc is 50, key is test_row_0/A:col10/1732371929313/Put/seqid=0 2024-11-23T14:25:30,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741962_1138 (size=12301) 2024-11-23T14:25:30,797 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-23T14:25:31,008 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/434b136c4a2d4b488a22f926be4e4e4d as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/434b136c4a2d4b488a22f926be4e4e4d 2024-11-23T14:25:31,008 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/ce365a2d62284ab49efafe69e2d61370 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/ce365a2d62284ab49efafe69e2d61370 2024-11-23T14:25:31,015 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5d1382f36edf9c126fa17e1f99c7f3fe/B of 5d1382f36edf9c126fa17e1f99c7f3fe into 434b136c4a2d4b488a22f926be4e4e4d(size=13.4 K), total size for store is 13.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:25:31,015 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5d1382f36edf9c126fa17e1f99c7f3fe/A of 5d1382f36edf9c126fa17e1f99c7f3fe into ce365a2d62284ab49efafe69e2d61370(size=13.4 K), total size for store is 13.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:25:31,015 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d1382f36edf9c126fa17e1f99c7f3fe: 2024-11-23T14:25:31,015 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d1382f36edf9c126fa17e1f99c7f3fe: 2024-11-23T14:25:31,015 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., storeName=5d1382f36edf9c126fa17e1f99c7f3fe/A, priority=13, startTime=1732371930583; duration=0sec 2024-11-23T14:25:31,015 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., storeName=5d1382f36edf9c126fa17e1f99c7f3fe/B, priority=13, startTime=1732371930583; duration=0sec 2024-11-23T14:25:31,016 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:25:31,016 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d1382f36edf9c126fa17e1f99c7f3fe:B 2024-11-23T14:25:31,016 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:25:31,016 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d1382f36edf9c126fa17e1f99c7f3fe:A 2024-11-23T14:25:31,016 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:25:31,017 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38197 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:25:31,018 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1541): 5d1382f36edf9c126fa17e1f99c7f3fe/C is initiating minor compaction (all files) 2024-11-23T14:25:31,018 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d1382f36edf9c126fa17e1f99c7f3fe/C in TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:31,018 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/4b1ad7c0bf0b49b2b36e897a310a2ab4, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/6e4616ba5294452b81278698593707a4, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/cc09233146984fc4924bc233e706d593] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp, totalSize=37.3 K 2024-11-23T14:25:31,018 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b1ad7c0bf0b49b2b36e897a310a2ab4, keycount=150, bloomtype=ROW, size=13.3 K, encoding=NONE, compression=NONE, seqNum=536, earliestPutTs=1732371927842 2024-11-23T14:25:31,019 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e4616ba5294452b81278698593707a4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=556, earliestPutTs=1732371928491 2024-11-23T14:25:31,019 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting cc09233146984fc4924bc233e706d593, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=579, earliestPutTs=1732371929270 2024-11-23T14:25:31,029 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d1382f36edf9c126fa17e1f99c7f3fe#C#compaction#126 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:25:31,029 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/58a6968bf0c24820989801ba569fe188 is 50, key is test_row_0/C:col10/1732371929273/Put/seqid=0 2024-11-23T14:25:31,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741963_1139 (size=13697) 2024-11-23T14:25:31,126 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=592 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/028f0115340740838d71dbb5b411c1bc 2024-11-23T14:25:31,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/4ed2ef953e7e40d1b94285d80ff1f4ad is 50, key is test_row_0/B:col10/1732371929313/Put/seqid=0 2024-11-23T14:25:31,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741964_1140 (size=12301) 2024-11-23T14:25:31,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on 5d1382f36edf9c126fa17e1f99c7f3fe 2024-11-23T14:25:31,414 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. as already flushing 2024-11-23T14:25:31,415 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-23T14:25:31,415 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T14:25:31,415 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:25:31,415 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:25:31,416 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T14:25:31,446 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/58a6968bf0c24820989801ba569fe188 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/58a6968bf0c24820989801ba569fe188 2024-11-23T14:25:31,452 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-23T14:25:31,452 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T14:25:31,452 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:25:31,453 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:25:31,453 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T14:25:31,454 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5d1382f36edf9c126fa17e1f99c7f3fe/C of 5d1382f36edf9c126fa17e1f99c7f3fe into 58a6968bf0c24820989801ba569fe188(size=13.4 K), total size for store is 13.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:25:31,454 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d1382f36edf9c126fa17e1f99c7f3fe: 2024-11-23T14:25:31,454 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., storeName=5d1382f36edf9c126fa17e1f99c7f3fe/C, priority=13, startTime=1732371930583; duration=0sec 2024-11-23T14:25:31,454 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:25:31,454 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d1382f36edf9c126fa17e1f99c7f3fe:C 2024-11-23T14:25:31,471 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-23T14:25:31,471 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T14:25:31,471 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:25:31,471 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:25:31,471 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T14:25:31,481 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-23T14:25:31,481 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-23T14:25:31,481 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T14:25:31,481 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T14:25:31,481 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:25:31,481 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:25:31,481 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:25:31,482 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:25:31,482 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T14:25:31,482 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T14:25:31,545 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=592 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/4ed2ef953e7e40d1b94285d80ff1f4ad 2024-11-23T14:25:31,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/5dbf07961ab14d1e9fdec158816d77ce is 50, key is test_row_0/C:col10/1732371929313/Put/seqid=0 2024-11-23T14:25:31,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741965_1141 (size=12301) 2024-11-23T14:25:31,962 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=592 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/5dbf07961ab14d1e9fdec158816d77ce 2024-11-23T14:25:31,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/028f0115340740838d71dbb5b411c1bc as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/028f0115340740838d71dbb5b411c1bc 2024-11-23T14:25:31,975 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/028f0115340740838d71dbb5b411c1bc, entries=150, sequenceid=592, filesize=12.0 K 2024-11-23T14:25:31,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/4ed2ef953e7e40d1b94285d80ff1f4ad as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/4ed2ef953e7e40d1b94285d80ff1f4ad 2024-11-23T14:25:31,981 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/4ed2ef953e7e40d1b94285d80ff1f4ad, entries=150, sequenceid=592, filesize=12.0 K 2024-11-23T14:25:31,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/5dbf07961ab14d1e9fdec158816d77ce as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/5dbf07961ab14d1e9fdec158816d77ce 2024-11-23T14:25:31,986 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/5dbf07961ab14d1e9fdec158816d77ce, entries=150, sequenceid=592, filesize=12.0 K 2024-11-23T14:25:31,987 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=33.54 KB/34350 for 5d1382f36edf9c126fa17e1f99c7f3fe in 1277ms, sequenceid=592, compaction requested=false 2024-11-23T14:25:31,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2603): Flush status journal for 5d1382f36edf9c126fa17e1f99c7f3fe: 2024-11-23T14:25:31,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:31,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=30 2024-11-23T14:25:31,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4169): Remote procedure done, pid=30 2024-11-23T14:25:31,989 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-23T14:25:31,989 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.4530 sec 2024-11-23T14:25:31,991 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=29, table=TestAcidGuarantees in 3.4590 sec 2024-11-23T14:25:32,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-23T14:25:32,690 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T14:25:32,690 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-23T14:25:32,691 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 73 2024-11-23T14:25:32,691 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 111 2024-11-23T14:25:32,691 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 75 2024-11-23T14:25:32,691 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 100 2024-11-23T14:25:32,691 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 111 2024-11-23T14:25:32,691 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-23T14:25:32,691 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3054 2024-11-23T14:25:32,691 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2955 2024-11-23T14:25:32,691 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-23T14:25:32,691 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1399 2024-11-23T14:25:32,691 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4193 rows 2024-11-23T14:25:32,691 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1400 2024-11-23T14:25:32,691 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4196 rows 2024-11-23T14:25:32,691 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-23T14:25:32,691 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T14:25:32,691 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:25:32,692 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:25:32,692 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T14:25:32,697 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-23T14:25:32,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-23T14:25:32,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-23T14:25:32,706 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732371932705"}]},"ts":"1732371932705"} 2024-11-23T14:25:32,708 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-23T14:25:32,708 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-23T14:25:32,709 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-23T14:25:32,713 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5d1382f36edf9c126fa17e1f99c7f3fe, UNASSIGN}] 2024-11-23T14:25:32,714 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5d1382f36edf9c126fa17e1f99c7f3fe, UNASSIGN 2024-11-23T14:25:32,730 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=5d1382f36edf9c126fa17e1f99c7f3fe, regionState=CLOSING, regionLocation=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:32,732 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5d1382f36edf9c126fa17e1f99c7f3fe, UNASSIGN because future has completed 2024-11-23T14:25:32,733 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-23T14:25:32,733 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767}] 2024-11-23T14:25:32,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-23T14:25:32,891 INFO [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(122): Close 5d1382f36edf9c126fa17e1f99c7f3fe 2024-11-23T14:25:32,891 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-23T14:25:32,892 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1722): Closing 5d1382f36edf9c126fa17e1f99c7f3fe, disabling compactions & flushes 2024-11-23T14:25:32,892 INFO [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:32,892 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:32,892 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. after waiting 0 ms 2024-11-23T14:25:32,892 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:32,892 INFO [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(2902): Flushing 5d1382f36edf9c126fa17e1f99c7f3fe 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-23T14:25:32,897 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/8e20e94de03d4f86ba091b6083d119c3 is 50, key is test_row_1/A:col10/1732371931480/Put/seqid=0 2024-11-23T14:25:32,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741966_1142 (size=9857) 2024-11-23T14:25:33,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-23T14:25:33,302 INFO [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=603 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/8e20e94de03d4f86ba091b6083d119c3 2024-11-23T14:25:33,315 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/82a8ddd7b94149a88e840f2b15da9292 is 50, key is test_row_1/B:col10/1732371931480/Put/seqid=0 2024-11-23T14:25:33,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741967_1143 (size=9857) 2024-11-23T14:25:33,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-23T14:25:33,721 INFO [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=603 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/82a8ddd7b94149a88e840f2b15da9292 2024-11-23T14:25:33,730 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/affdce1a53ab4cb5a9ef9386c7e1d4c8 is 50, key is test_row_1/C:col10/1732371931480/Put/seqid=0 2024-11-23T14:25:33,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741968_1144 (size=9857) 2024-11-23T14:25:33,734 INFO [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=603 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/affdce1a53ab4cb5a9ef9386c7e1d4c8 2024-11-23T14:25:33,741 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/A/8e20e94de03d4f86ba091b6083d119c3 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/8e20e94de03d4f86ba091b6083d119c3 2024-11-23T14:25:33,746 INFO [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/8e20e94de03d4f86ba091b6083d119c3, entries=100, sequenceid=603, filesize=9.6 K 2024-11-23T14:25:33,747 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/B/82a8ddd7b94149a88e840f2b15da9292 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/82a8ddd7b94149a88e840f2b15da9292 2024-11-23T14:25:33,752 INFO [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/82a8ddd7b94149a88e840f2b15da9292, entries=100, sequenceid=603, filesize=9.6 K 2024-11-23T14:25:33,753 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/.tmp/C/affdce1a53ab4cb5a9ef9386c7e1d4c8 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/affdce1a53ab4cb5a9ef9386c7e1d4c8 2024-11-23T14:25:33,759 INFO [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/affdce1a53ab4cb5a9ef9386c7e1d4c8, entries=100, sequenceid=603, filesize=9.6 K 2024-11-23T14:25:33,760 INFO [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 5d1382f36edf9c126fa17e1f99c7f3fe in 868ms, sequenceid=603, compaction requested=true 2024-11-23T14:25:33,761 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/2f3495313ddc4d57a58cc9f9f8057c51, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/be37907be43e42c4a5cc28e8ade494a8, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/36a49bff398744afa17989f1918bc895, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/130a7944e2004a1a8cc64d56b2a61d9f, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/062b372f06be4727bc2b62598dbbbfcb, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/5fdb8e01052c41a8a692ff3ee5ffebda, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/7a0fcdf1079246b3aab768384e61dba6, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/9f6fe0ce224941f083175f80ff7d67ed, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/e34d3a2381944f4b9f56ec81f334cd2c, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/0489d8084889467c922077b874ea49b9, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/7e2afbcba7604e4c8e6371854744ad95, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/84b54d10afc74de69276be79ea1b3d6f, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/57f7b2cae40f44c5af9b2f3ff8bd9fe4, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/87877e94345e4534a978a8c207bff523, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/1705333fb04e4a4787d9994f56ac0c53, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/03a0f119ad79421d9c2c5b1448df2211, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/ff53e25d6541408c89424a2aa6a76fed, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/34163857dc9449ceace9d451fb26d035, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/ae318f68ac7b463c9e61df53210a72a5, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/1e9fdc3213164ee9a757f27953fd6c3e, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/ecd3b772d7cd4c938f8b754719ad7362, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/ce831b7a1e9b49bc91b50cd8acaf5111, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/48c220a788e246b6bda49c92d1ae3935, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/dfa2c4c4ae29490eb65becaf574f3087, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/7f09fd65de4844888f4a9d9055ea805a, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/710d99614bd847bea626db0412f8abd2, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/49ad3fbc31e44f19bf88c47d8f0dcaf9, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/594d187c4ea743e48263be15095bfdce, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/13362274105c4039a1c2457f499e8bc4, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/d7fb76d3b4264c7998dbc2566cf97835, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/731630046f7c44a099a0aff9f3ed425d, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/c0e63ebaf2b1431d98d8b5b471b9fef2, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/3cf04e56c7a849e18ef8175ff57427b3, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/145a0fb715cf4e7d9a92964582b7398e, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/0b877fae18ed4b9faa30a2ae34bd1826, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/0e9ed7cdc8da43b88ff151bea1ccd7a9, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/41c6b432b77c4512bb2f99030ec78167, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/58f2638db90c434cb0dcd300494d37d3, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/7c9308a724f7479ab0af528a5e8cb484, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/49f73e9539584f6cbff3fcc5bb888d2d, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/13a63584e98343ad8fc150995c7b116f] to archive 2024-11-23T14:25:33,764 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-23T14:25:33,770 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/2f3495313ddc4d57a58cc9f9f8057c51 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/2f3495313ddc4d57a58cc9f9f8057c51 2024-11-23T14:25:33,773 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/be37907be43e42c4a5cc28e8ade494a8 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/be37907be43e42c4a5cc28e8ade494a8 2024-11-23T14:25:33,775 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/36a49bff398744afa17989f1918bc895 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/36a49bff398744afa17989f1918bc895 2024-11-23T14:25:33,777 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/130a7944e2004a1a8cc64d56b2a61d9f to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/130a7944e2004a1a8cc64d56b2a61d9f 2024-11-23T14:25:33,779 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/062b372f06be4727bc2b62598dbbbfcb to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/062b372f06be4727bc2b62598dbbbfcb 2024-11-23T14:25:33,781 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/5fdb8e01052c41a8a692ff3ee5ffebda to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/5fdb8e01052c41a8a692ff3ee5ffebda 2024-11-23T14:25:33,783 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/7a0fcdf1079246b3aab768384e61dba6 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/7a0fcdf1079246b3aab768384e61dba6 2024-11-23T14:25:33,785 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/9f6fe0ce224941f083175f80ff7d67ed to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/9f6fe0ce224941f083175f80ff7d67ed 2024-11-23T14:25:33,786 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/e34d3a2381944f4b9f56ec81f334cd2c to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/e34d3a2381944f4b9f56ec81f334cd2c 2024-11-23T14:25:33,789 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/0489d8084889467c922077b874ea49b9 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/0489d8084889467c922077b874ea49b9 2024-11-23T14:25:33,790 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/7e2afbcba7604e4c8e6371854744ad95 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/7e2afbcba7604e4c8e6371854744ad95 2024-11-23T14:25:33,792 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/84b54d10afc74de69276be79ea1b3d6f to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/84b54d10afc74de69276be79ea1b3d6f 2024-11-23T14:25:33,793 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/57f7b2cae40f44c5af9b2f3ff8bd9fe4 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/57f7b2cae40f44c5af9b2f3ff8bd9fe4 2024-11-23T14:25:33,794 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/87877e94345e4534a978a8c207bff523 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/87877e94345e4534a978a8c207bff523 2024-11-23T14:25:33,797 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/1705333fb04e4a4787d9994f56ac0c53 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/1705333fb04e4a4787d9994f56ac0c53 2024-11-23T14:25:33,799 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/03a0f119ad79421d9c2c5b1448df2211 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/03a0f119ad79421d9c2c5b1448df2211 2024-11-23T14:25:33,801 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/ff53e25d6541408c89424a2aa6a76fed to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/ff53e25d6541408c89424a2aa6a76fed 2024-11-23T14:25:33,803 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/34163857dc9449ceace9d451fb26d035 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/34163857dc9449ceace9d451fb26d035 2024-11-23T14:25:33,805 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/ae318f68ac7b463c9e61df53210a72a5 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/ae318f68ac7b463c9e61df53210a72a5 2024-11-23T14:25:33,806 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/1e9fdc3213164ee9a757f27953fd6c3e to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/1e9fdc3213164ee9a757f27953fd6c3e 2024-11-23T14:25:33,808 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/ecd3b772d7cd4c938f8b754719ad7362 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/ecd3b772d7cd4c938f8b754719ad7362 2024-11-23T14:25:33,810 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/ce831b7a1e9b49bc91b50cd8acaf5111 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/ce831b7a1e9b49bc91b50cd8acaf5111 2024-11-23T14:25:33,812 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/48c220a788e246b6bda49c92d1ae3935 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/48c220a788e246b6bda49c92d1ae3935 2024-11-23T14:25:33,815 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/dfa2c4c4ae29490eb65becaf574f3087 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/dfa2c4c4ae29490eb65becaf574f3087 2024-11-23T14:25:33,817 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/7f09fd65de4844888f4a9d9055ea805a to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/7f09fd65de4844888f4a9d9055ea805a 2024-11-23T14:25:33,819 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/710d99614bd847bea626db0412f8abd2 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/710d99614bd847bea626db0412f8abd2 2024-11-23T14:25:33,820 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/49ad3fbc31e44f19bf88c47d8f0dcaf9 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/49ad3fbc31e44f19bf88c47d8f0dcaf9 2024-11-23T14:25:33,822 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/594d187c4ea743e48263be15095bfdce to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/594d187c4ea743e48263be15095bfdce 2024-11-23T14:25:33,824 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/13362274105c4039a1c2457f499e8bc4 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/13362274105c4039a1c2457f499e8bc4 2024-11-23T14:25:33,826 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/d7fb76d3b4264c7998dbc2566cf97835 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/d7fb76d3b4264c7998dbc2566cf97835 2024-11-23T14:25:33,828 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/731630046f7c44a099a0aff9f3ed425d to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/731630046f7c44a099a0aff9f3ed425d 2024-11-23T14:25:33,830 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/c0e63ebaf2b1431d98d8b5b471b9fef2 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/c0e63ebaf2b1431d98d8b5b471b9fef2 2024-11-23T14:25:33,831 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/3cf04e56c7a849e18ef8175ff57427b3 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/3cf04e56c7a849e18ef8175ff57427b3 2024-11-23T14:25:33,833 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/145a0fb715cf4e7d9a92964582b7398e to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/145a0fb715cf4e7d9a92964582b7398e 2024-11-23T14:25:33,834 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/0b877fae18ed4b9faa30a2ae34bd1826 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/0b877fae18ed4b9faa30a2ae34bd1826 2024-11-23T14:25:33,835 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/0e9ed7cdc8da43b88ff151bea1ccd7a9 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/0e9ed7cdc8da43b88ff151bea1ccd7a9 2024-11-23T14:25:33,837 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/41c6b432b77c4512bb2f99030ec78167 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/41c6b432b77c4512bb2f99030ec78167 2024-11-23T14:25:33,838 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/58f2638db90c434cb0dcd300494d37d3 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/58f2638db90c434cb0dcd300494d37d3 2024-11-23T14:25:33,840 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/7c9308a724f7479ab0af528a5e8cb484 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/7c9308a724f7479ab0af528a5e8cb484 2024-11-23T14:25:33,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-23T14:25:33,841 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/49f73e9539584f6cbff3fcc5bb888d2d to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/49f73e9539584f6cbff3fcc5bb888d2d 2024-11-23T14:25:33,843 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/13a63584e98343ad8fc150995c7b116f to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/13a63584e98343ad8fc150995c7b116f 2024-11-23T14:25:33,862 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/59f43f9d881248d5891b77dd7aa0a3ed, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/a804626854494ff782430763fdc6e74d, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/e2ed9146f9c644d197142d726542a25e, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/454815fae1124de5970090810f0634f0, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/e8caa35907ed427fa90bea148eb97606, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/6098ef6602ad46bf962d5d4be1d40884, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/dea0eabfc5094034865beb8d2ebc7f68, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/65c6b76d14dc4f23867d16e05c184798, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/1f528556cc714ccab2ed6496fa356ae9, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/944e31039d4a444cafbeee6ed03b1653, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/8d538adcc3c54b99a55ef245f5f7b7de, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/09fcb7ca83284ccebbe9b6da9173fdfd, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/24d7f28c7ae243b5b1f2e38e2ba5b310, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/16ef9a7f6de64f5ead7d2738482a1ae7, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/a2e096fc89f94798a77fe991d139b779, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/4b12670ef5af427ca84413300855af6c, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/85bb31a16194456fa4ce30d5fc61c1f3, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/ea84ec993cf14f06a2b46aefdf59d440, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/c26f740ff82c4769a94e0d2dc68800e9, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/21db5a459f834d16b18f027916db3319, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/1d841a4d67e14443be84c26c65fc64ae, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/f7e3f2529ae04c09a4e39818d31df6e1, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/0715c60bb3054fbdaacfaa5b2d61529d, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/fca49e1457244240b9226612c74a615e, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/2142bf98f443484689941c1b3fa19bba, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/3971d71c698e41acab7a199402c6aeff, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/36bdc8aaa0a54d2197f056b414ac6761, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/8fab80edcc3d4988b669c0c06e4cad6e, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/e998814dd4f84b939003d89c5c1ed246, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/7f07a212d1a848568096f1af7f97846f, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/86ecbce2eb984e4b965163544209d960, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/dfc43dcd11c44bdd8728a1127f72b585, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/1f1a7e673ea84275987ecbd1c6683b00, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/e7bc5542481b4b9eaf0ac7a7c2459711, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/3db686dd3cc74988802cb13fef50854f, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/5df6151be6ce44ab808136c8ef2b1259, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/0be7e18aaba84fb3bd9c626668e3eed9, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/ec61aecc634042a4af3040b0d7490cd8, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/a6ff97e205c04b70951cfb19ce41fd2c, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/37ac88b1bf3a46aaac1274c9936914c9, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/f37b505933bd48a1ae56d66145e0f6bf] to archive 2024-11-23T14:25:33,863 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-23T14:25:33,865 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/59f43f9d881248d5891b77dd7aa0a3ed to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/59f43f9d881248d5891b77dd7aa0a3ed 2024-11-23T14:25:33,867 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/a804626854494ff782430763fdc6e74d to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/a804626854494ff782430763fdc6e74d 2024-11-23T14:25:33,868 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/e2ed9146f9c644d197142d726542a25e to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/e2ed9146f9c644d197142d726542a25e 2024-11-23T14:25:33,870 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/454815fae1124de5970090810f0634f0 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/454815fae1124de5970090810f0634f0 2024-11-23T14:25:33,872 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/e8caa35907ed427fa90bea148eb97606 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/e8caa35907ed427fa90bea148eb97606 2024-11-23T14:25:33,874 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/6098ef6602ad46bf962d5d4be1d40884 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/6098ef6602ad46bf962d5d4be1d40884 2024-11-23T14:25:33,875 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/dea0eabfc5094034865beb8d2ebc7f68 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/dea0eabfc5094034865beb8d2ebc7f68 2024-11-23T14:25:33,877 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/65c6b76d14dc4f23867d16e05c184798 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/65c6b76d14dc4f23867d16e05c184798 2024-11-23T14:25:33,878 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/1f528556cc714ccab2ed6496fa356ae9 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/1f528556cc714ccab2ed6496fa356ae9 2024-11-23T14:25:33,880 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/944e31039d4a444cafbeee6ed03b1653 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/944e31039d4a444cafbeee6ed03b1653 2024-11-23T14:25:33,881 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/8d538adcc3c54b99a55ef245f5f7b7de to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/8d538adcc3c54b99a55ef245f5f7b7de 2024-11-23T14:25:33,883 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/09fcb7ca83284ccebbe9b6da9173fdfd to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/09fcb7ca83284ccebbe9b6da9173fdfd 2024-11-23T14:25:33,884 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/24d7f28c7ae243b5b1f2e38e2ba5b310 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/24d7f28c7ae243b5b1f2e38e2ba5b310 2024-11-23T14:25:33,886 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/16ef9a7f6de64f5ead7d2738482a1ae7 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/16ef9a7f6de64f5ead7d2738482a1ae7 2024-11-23T14:25:33,887 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/a2e096fc89f94798a77fe991d139b779 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/a2e096fc89f94798a77fe991d139b779 2024-11-23T14:25:33,888 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/4b12670ef5af427ca84413300855af6c to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/4b12670ef5af427ca84413300855af6c 2024-11-23T14:25:33,890 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/85bb31a16194456fa4ce30d5fc61c1f3 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/85bb31a16194456fa4ce30d5fc61c1f3 2024-11-23T14:25:33,891 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/ea84ec993cf14f06a2b46aefdf59d440 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/ea84ec993cf14f06a2b46aefdf59d440 2024-11-23T14:25:33,892 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/c26f740ff82c4769a94e0d2dc68800e9 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/c26f740ff82c4769a94e0d2dc68800e9 2024-11-23T14:25:33,894 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/21db5a459f834d16b18f027916db3319 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/21db5a459f834d16b18f027916db3319 2024-11-23T14:25:33,895 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/1d841a4d67e14443be84c26c65fc64ae to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/1d841a4d67e14443be84c26c65fc64ae 2024-11-23T14:25:33,896 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/f7e3f2529ae04c09a4e39818d31df6e1 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/f7e3f2529ae04c09a4e39818d31df6e1 2024-11-23T14:25:33,898 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/0715c60bb3054fbdaacfaa5b2d61529d to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/0715c60bb3054fbdaacfaa5b2d61529d 2024-11-23T14:25:33,899 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/fca49e1457244240b9226612c74a615e to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/fca49e1457244240b9226612c74a615e 2024-11-23T14:25:33,900 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/2142bf98f443484689941c1b3fa19bba to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/2142bf98f443484689941c1b3fa19bba 2024-11-23T14:25:33,902 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/3971d71c698e41acab7a199402c6aeff to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/3971d71c698e41acab7a199402c6aeff 2024-11-23T14:25:33,903 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/36bdc8aaa0a54d2197f056b414ac6761 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/36bdc8aaa0a54d2197f056b414ac6761 2024-11-23T14:25:33,904 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/8fab80edcc3d4988b669c0c06e4cad6e to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/8fab80edcc3d4988b669c0c06e4cad6e 2024-11-23T14:25:33,906 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/e998814dd4f84b939003d89c5c1ed246 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/e998814dd4f84b939003d89c5c1ed246 2024-11-23T14:25:33,907 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/7f07a212d1a848568096f1af7f97846f to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/7f07a212d1a848568096f1af7f97846f 2024-11-23T14:25:33,911 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/86ecbce2eb984e4b965163544209d960 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/86ecbce2eb984e4b965163544209d960 2024-11-23T14:25:33,913 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/dfc43dcd11c44bdd8728a1127f72b585 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/dfc43dcd11c44bdd8728a1127f72b585 2024-11-23T14:25:33,918 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/1f1a7e673ea84275987ecbd1c6683b00 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/1f1a7e673ea84275987ecbd1c6683b00 2024-11-23T14:25:33,920 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/e7bc5542481b4b9eaf0ac7a7c2459711 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/e7bc5542481b4b9eaf0ac7a7c2459711 2024-11-23T14:25:33,922 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/3db686dd3cc74988802cb13fef50854f to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/3db686dd3cc74988802cb13fef50854f 2024-11-23T14:25:33,923 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/5df6151be6ce44ab808136c8ef2b1259 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/5df6151be6ce44ab808136c8ef2b1259 2024-11-23T14:25:33,925 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/0be7e18aaba84fb3bd9c626668e3eed9 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/0be7e18aaba84fb3bd9c626668e3eed9 2024-11-23T14:25:33,926 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/ec61aecc634042a4af3040b0d7490cd8 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/ec61aecc634042a4af3040b0d7490cd8 2024-11-23T14:25:33,928 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/a6ff97e205c04b70951cfb19ce41fd2c to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/a6ff97e205c04b70951cfb19ce41fd2c 2024-11-23T14:25:33,929 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/37ac88b1bf3a46aaac1274c9936914c9 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/37ac88b1bf3a46aaac1274c9936914c9 2024-11-23T14:25:33,931 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/f37b505933bd48a1ae56d66145e0f6bf to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/f37b505933bd48a1ae56d66145e0f6bf 2024-11-23T14:25:33,934 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/00f2e7c3be6f4d0d914874f9671f60bf, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/dbd269da3b0b42d5891afedba4052f31, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/913deceb7dfd420b8ea526bc5e0c3aab, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/bde64bd3db344bffbf0eb6c6a52fcdb9, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/8b7949c65c0e4b9789789962f16f884c, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/4af6fca728f64c45bce2537b306ad802, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/909fb2e98d934d6aa4cfec4332bbf270, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/aa4b01178fd544d38c96b6a4ad6cc4b6, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/a54913e8bc234be080d5ecf6c488c686, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/291dc2ca16aa4a8b85e25bbdd0b44025, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/c288c7a7b1db45dc9ea529338012354e, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/3ef064e14c8b42a4a677b25d545cc65a, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/708d9252368d425181330c0696410e23, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/82538cafb29f4d2389e1b492fc913de4, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/f142299baa26466bb7c6d7f43e4cccd4, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/f26f8853a26e452289bd01f69b5d29c3, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/6d2d95b572e24f24ba3762810dafcc3b, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/d5656e6d9746440497424aedd94656d5, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/d1b8d962bea746a0974827bfa5cafbea, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/c85a9811fd5841959f31e68d5a073372, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/400fa77350604812a06daf40ca850362, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/06455c05b7d045aa9b94b58a3acf1a3a, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/b58122625945402087e47e6300a0e980, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/b1679cc41af34aa3a962fa5ae5252479, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/8ce659ccc96245c79383fac2f0a30f20, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/e9870db655984b7ba06bb5c72d4e08a8, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/aec993c358554e5a8781d73497d17c9d, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/d22071479c8d4828b17c68941c6cf521, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/cdb1814505a94b87882fc80cb8482317, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/99db463312e841d091d23483579fd35e, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/f1a5ac9f8c1941eaa251fea32d59e1d4, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/1414e1a990c0450a89e890cccf2e6678, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/62a23aa4fb5f4a82866df520d2709cde, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/acc24f8aa9054f0089af4c19ca9b8ed9, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/62cb55ba63c248ad86fb2f6a21abc48d, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/466af783f01544229a38f9e94eba363a, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/f95e6348c8da4e3f8d53e862e0b8148b, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/4b1ad7c0bf0b49b2b36e897a310a2ab4, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/b2157d6fd72445a88ac8a4df6bf2ee6f, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/6e4616ba5294452b81278698593707a4, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/cc09233146984fc4924bc233e706d593] to archive 2024-11-23T14:25:33,935 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-23T14:25:33,937 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/00f2e7c3be6f4d0d914874f9671f60bf to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/00f2e7c3be6f4d0d914874f9671f60bf 2024-11-23T14:25:33,938 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/dbd269da3b0b42d5891afedba4052f31 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/dbd269da3b0b42d5891afedba4052f31 2024-11-23T14:25:33,940 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/913deceb7dfd420b8ea526bc5e0c3aab to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/913deceb7dfd420b8ea526bc5e0c3aab 2024-11-23T14:25:33,942 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/bde64bd3db344bffbf0eb6c6a52fcdb9 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/bde64bd3db344bffbf0eb6c6a52fcdb9 2024-11-23T14:25:33,944 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/8b7949c65c0e4b9789789962f16f884c to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/8b7949c65c0e4b9789789962f16f884c 2024-11-23T14:25:33,945 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/4af6fca728f64c45bce2537b306ad802 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/4af6fca728f64c45bce2537b306ad802 2024-11-23T14:25:33,947 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/909fb2e98d934d6aa4cfec4332bbf270 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/909fb2e98d934d6aa4cfec4332bbf270 2024-11-23T14:25:33,948 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/aa4b01178fd544d38c96b6a4ad6cc4b6 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/aa4b01178fd544d38c96b6a4ad6cc4b6 2024-11-23T14:25:33,950 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/a54913e8bc234be080d5ecf6c488c686 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/a54913e8bc234be080d5ecf6c488c686 2024-11-23T14:25:33,951 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/291dc2ca16aa4a8b85e25bbdd0b44025 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/291dc2ca16aa4a8b85e25bbdd0b44025 2024-11-23T14:25:33,952 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/c288c7a7b1db45dc9ea529338012354e to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/c288c7a7b1db45dc9ea529338012354e 2024-11-23T14:25:33,953 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/3ef064e14c8b42a4a677b25d545cc65a to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/3ef064e14c8b42a4a677b25d545cc65a 2024-11-23T14:25:33,956 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/708d9252368d425181330c0696410e23 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/708d9252368d425181330c0696410e23 2024-11-23T14:25:33,958 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/82538cafb29f4d2389e1b492fc913de4 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/82538cafb29f4d2389e1b492fc913de4 2024-11-23T14:25:33,960 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/f142299baa26466bb7c6d7f43e4cccd4 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/f142299baa26466bb7c6d7f43e4cccd4 2024-11-23T14:25:33,962 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/f26f8853a26e452289bd01f69b5d29c3 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/f26f8853a26e452289bd01f69b5d29c3 2024-11-23T14:25:33,964 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/6d2d95b572e24f24ba3762810dafcc3b to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/6d2d95b572e24f24ba3762810dafcc3b 2024-11-23T14:25:33,966 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/d5656e6d9746440497424aedd94656d5 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/d5656e6d9746440497424aedd94656d5 2024-11-23T14:25:33,967 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/d1b8d962bea746a0974827bfa5cafbea to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/d1b8d962bea746a0974827bfa5cafbea 2024-11-23T14:25:33,968 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/c85a9811fd5841959f31e68d5a073372 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/c85a9811fd5841959f31e68d5a073372 2024-11-23T14:25:33,970 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/400fa77350604812a06daf40ca850362 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/400fa77350604812a06daf40ca850362 2024-11-23T14:25:33,971 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/06455c05b7d045aa9b94b58a3acf1a3a to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/06455c05b7d045aa9b94b58a3acf1a3a 2024-11-23T14:25:33,972 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/b58122625945402087e47e6300a0e980 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/b58122625945402087e47e6300a0e980 2024-11-23T14:25:33,974 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/b1679cc41af34aa3a962fa5ae5252479 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/b1679cc41af34aa3a962fa5ae5252479 2024-11-23T14:25:33,975 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/8ce659ccc96245c79383fac2f0a30f20 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/8ce659ccc96245c79383fac2f0a30f20 2024-11-23T14:25:33,977 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/e9870db655984b7ba06bb5c72d4e08a8 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/e9870db655984b7ba06bb5c72d4e08a8 2024-11-23T14:25:33,979 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/aec993c358554e5a8781d73497d17c9d to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/aec993c358554e5a8781d73497d17c9d 2024-11-23T14:25:33,980 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/d22071479c8d4828b17c68941c6cf521 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/d22071479c8d4828b17c68941c6cf521 2024-11-23T14:25:33,983 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/cdb1814505a94b87882fc80cb8482317 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/cdb1814505a94b87882fc80cb8482317 2024-11-23T14:25:33,986 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/99db463312e841d091d23483579fd35e to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/99db463312e841d091d23483579fd35e 2024-11-23T14:25:33,988 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/f1a5ac9f8c1941eaa251fea32d59e1d4 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/f1a5ac9f8c1941eaa251fea32d59e1d4 2024-11-23T14:25:33,990 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/1414e1a990c0450a89e890cccf2e6678 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/1414e1a990c0450a89e890cccf2e6678 2024-11-23T14:25:33,992 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/62a23aa4fb5f4a82866df520d2709cde to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/62a23aa4fb5f4a82866df520d2709cde 2024-11-23T14:25:33,994 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/acc24f8aa9054f0089af4c19ca9b8ed9 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/acc24f8aa9054f0089af4c19ca9b8ed9 2024-11-23T14:25:33,996 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/62cb55ba63c248ad86fb2f6a21abc48d to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/62cb55ba63c248ad86fb2f6a21abc48d 2024-11-23T14:25:33,998 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/466af783f01544229a38f9e94eba363a to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/466af783f01544229a38f9e94eba363a 2024-11-23T14:25:33,999 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/f95e6348c8da4e3f8d53e862e0b8148b to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/f95e6348c8da4e3f8d53e862e0b8148b 2024-11-23T14:25:34,001 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/4b1ad7c0bf0b49b2b36e897a310a2ab4 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/4b1ad7c0bf0b49b2b36e897a310a2ab4 2024-11-23T14:25:34,003 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/b2157d6fd72445a88ac8a4df6bf2ee6f to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/b2157d6fd72445a88ac8a4df6bf2ee6f 2024-11-23T14:25:34,005 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/6e4616ba5294452b81278698593707a4 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/6e4616ba5294452b81278698593707a4 2024-11-23T14:25:34,007 DEBUG [StoreCloser-TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/cc09233146984fc4924bc233e706d593 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/cc09233146984fc4924bc233e706d593 2024-11-23T14:25:34,016 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/recovered.edits/606.seqid, newMaxSeqId=606, maxSeqId=1 2024-11-23T14:25:34,050 INFO [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. 2024-11-23T14:25:34,050 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1676): Region close journal for 5d1382f36edf9c126fa17e1f99c7f3fe: Waiting for close lock at 1732371932892Running coprocessor pre-close hooks at 1732371932892Disabling compacts and flushes for region at 1732371932892Disabling writes for close at 1732371932892Obtaining lock to block concurrent updates at 1732371932892Preparing flush snapshotting stores in 5d1382f36edf9c126fa17e1f99c7f3fe at 1732371932892Finished memstore snapshotting TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1732371932893 (+1 ms)Flushing stores of TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe. at 1732371932893Flushing 5d1382f36edf9c126fa17e1f99c7f3fe/A: creating writer at 1732371932893Flushing 5d1382f36edf9c126fa17e1f99c7f3fe/A: appending metadata at 1732371932897 (+4 ms)Flushing 5d1382f36edf9c126fa17e1f99c7f3fe/A: closing flushed file at 1732371932897Flushing 5d1382f36edf9c126fa17e1f99c7f3fe/B: creating writer at 1732371933310 (+413 ms)Flushing 5d1382f36edf9c126fa17e1f99c7f3fe/B: appending metadata at 1732371933314 (+4 ms)Flushing 5d1382f36edf9c126fa17e1f99c7f3fe/B: closing flushed file at 1732371933315 (+1 ms)Flushing 5d1382f36edf9c126fa17e1f99c7f3fe/C: creating writer at 1732371933726 (+411 ms)Flushing 5d1382f36edf9c126fa17e1f99c7f3fe/C: appending metadata at 1732371933729 (+3 ms)Flushing 5d1382f36edf9c126fa17e1f99c7f3fe/C: closing flushed file at 1732371933729Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@440b79f0: reopening flushed file at 1732371933739 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1f1f1191: reopening flushed file at 1732371933747 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@206b1abd: reopening flushed file at 1732371933752 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 5d1382f36edf9c126fa17e1f99c7f3fe in 868ms, sequenceid=603, compaction requested=true at 1732371933760 (+8 ms)Writing region close event to WAL at 1732371934011 (+251 ms)Running coprocessor post-close hooks at 1732371934049 (+38 ms)Closed at 1732371934049 2024-11-23T14:25:34,053 INFO [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(157): Closed 5d1382f36edf9c126fa17e1f99c7f3fe 2024-11-23T14:25:34,054 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=5d1382f36edf9c126fa17e1f99c7f3fe, regionState=CLOSED 2024-11-23T14:25:34,056 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 because future has completed 2024-11-23T14:25:34,059 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-23T14:25:34,059 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 5d1382f36edf9c126fa17e1f99c7f3fe, server=c6f7b1897123,35385,1732371902767 in 1.3240 sec 2024-11-23T14:25:34,061 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-23T14:25:34,062 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5d1382f36edf9c126fa17e1f99c7f3fe, UNASSIGN in 1.3460 sec 2024-11-23T14:25:34,065 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-23T14:25:34,065 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.3540 sec 2024-11-23T14:25:34,067 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732371934067"}]},"ts":"1732371934067"} 2024-11-23T14:25:34,068 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-23T14:25:34,068 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-23T14:25:34,070 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.3700 sec 2024-11-23T14:25:34,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-23T14:25:34,851 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-23T14:25:34,856 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-23T14:25:34,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] procedure2.ProcedureExecutor(1139): Stored pid=35, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T14:25:34,864 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=35, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T14:25:34,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=35 2024-11-23T14:25:34,865 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=35, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T14:25:34,870 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe 2024-11-23T14:25:34,875 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A, FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B, FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C, FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/recovered.edits] 2024-11-23T14:25:34,878 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/028f0115340740838d71dbb5b411c1bc to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/028f0115340740838d71dbb5b411c1bc 2024-11-23T14:25:34,879 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/8e20e94de03d4f86ba091b6083d119c3 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/8e20e94de03d4f86ba091b6083d119c3 2024-11-23T14:25:34,880 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/ce365a2d62284ab49efafe69e2d61370 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/A/ce365a2d62284ab49efafe69e2d61370 2024-11-23T14:25:34,882 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/434b136c4a2d4b488a22f926be4e4e4d to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/434b136c4a2d4b488a22f926be4e4e4d 2024-11-23T14:25:34,884 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/4ed2ef953e7e40d1b94285d80ff1f4ad to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/4ed2ef953e7e40d1b94285d80ff1f4ad 2024-11-23T14:25:34,885 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/82a8ddd7b94149a88e840f2b15da9292 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/B/82a8ddd7b94149a88e840f2b15da9292 2024-11-23T14:25:34,888 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/58a6968bf0c24820989801ba569fe188 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/58a6968bf0c24820989801ba569fe188 2024-11-23T14:25:34,889 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/5dbf07961ab14d1e9fdec158816d77ce to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/5dbf07961ab14d1e9fdec158816d77ce 2024-11-23T14:25:34,891 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/affdce1a53ab4cb5a9ef9386c7e1d4c8 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/C/affdce1a53ab4cb5a9ef9386c7e1d4c8 2024-11-23T14:25:34,894 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/recovered.edits/606.seqid to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe/recovered.edits/606.seqid 2024-11-23T14:25:34,894 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/5d1382f36edf9c126fa17e1f99c7f3fe 2024-11-23T14:25:34,895 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-23T14:25:34,899 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=35, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T14:25:34,903 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-23T14:25:34,908 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-23T14:25:34,936 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-23T14:25:34,937 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=35, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T14:25:34,937 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-23T14:25:34,938 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732371934937"}]},"ts":"9223372036854775807"} 2024-11-23T14:25:34,942 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-23T14:25:34,942 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 5d1382f36edf9c126fa17e1f99c7f3fe, NAME => 'TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe.', STARTKEY => '', ENDKEY => ''}] 2024-11-23T14:25:34,942 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-23T14:25:34,943 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732371934943"}]},"ts":"9223372036854775807"} 2024-11-23T14:25:34,946 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-23T14:25:34,947 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=35, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T14:25:34,948 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 89 msec 2024-11-23T14:25:34,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=35 2024-11-23T14:25:34,971 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-23T14:25:34,971 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-23T14:25:34,985 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=239 (was 211) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_412678638_22 at /127.0.0.1:50000 [Waiting for operation #47] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_412678638_22 at /127.0.0.1:41298 [Waiting for operation #381] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_107099228_22 at /127.0.0.1:41150 [Waiting for operation #394] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;c6f7b1897123:35385-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=457 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=507 (was 426) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3796 (was 3716) - AvailableMemoryMB LEAK? - 2024-11-23T14:25:34,995 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=239, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=507, ProcessCount=11, AvailableMemoryMB=3795 2024-11-23T14:25:34,996 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-23T14:25:34,997 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-23T14:25:34,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] procedure2.ProcedureExecutor(1139): Stored pid=36, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-23T14:25:34,999 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-23T14:25:35,000 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:35,000 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 36 2024-11-23T14:25:35,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-23T14:25:35,001 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-23T14:25:35,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741969_1145 (size=959) 2024-11-23T14:25:35,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-23T14:25:35,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-23T14:25:35,411 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a 2024-11-23T14:25:35,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741970_1146 (size=53) 2024-11-23T14:25:35,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-23T14:25:35,817 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T14:25:35,817 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing a36a6f46a23b82a88d429516318dcd05, disabling compactions & flushes 2024-11-23T14:25:35,817 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:35,818 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:35,818 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. after waiting 0 ms 2024-11-23T14:25:35,818 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:35,818 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:35,818 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for a36a6f46a23b82a88d429516318dcd05: Waiting for close lock at 1732371935817Disabling compacts and flushes for region at 1732371935817Disabling writes for close at 1732371935818 (+1 ms)Writing region close event to WAL at 1732371935818Closed at 1732371935818 2024-11-23T14:25:35,819 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-23T14:25:35,819 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732371935819"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732371935819"}]},"ts":"1732371935819"} 2024-11-23T14:25:35,822 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-23T14:25:35,822 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-23T14:25:35,823 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732371935822"}]},"ts":"1732371935822"} 2024-11-23T14:25:35,824 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-23T14:25:35,825 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a36a6f46a23b82a88d429516318dcd05, ASSIGN}] 2024-11-23T14:25:35,826 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a36a6f46a23b82a88d429516318dcd05, ASSIGN 2024-11-23T14:25:35,827 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a36a6f46a23b82a88d429516318dcd05, ASSIGN; state=OFFLINE, location=c6f7b1897123,35385,1732371902767; forceNewPlan=false, retain=false 2024-11-23T14:25:35,978 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=a36a6f46a23b82a88d429516318dcd05, regionState=OPENING, regionLocation=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:35,982 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a36a6f46a23b82a88d429516318dcd05, ASSIGN because future has completed 2024-11-23T14:25:35,983 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767}] 2024-11-23T14:25:36,139 INFO [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:36,139 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7752): Opening region: {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} 2024-11-23T14:25:36,139 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees a36a6f46a23b82a88d429516318dcd05 2024-11-23T14:25:36,139 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T14:25:36,139 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7794): checking encryption for a36a6f46a23b82a88d429516318dcd05 2024-11-23T14:25:36,139 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7797): checking classloading for a36a6f46a23b82a88d429516318dcd05 2024-11-23T14:25:36,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-23T14:25:36,141 INFO [StoreOpener-a36a6f46a23b82a88d429516318dcd05-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region a36a6f46a23b82a88d429516318dcd05 2024-11-23T14:25:36,143 INFO [StoreOpener-a36a6f46a23b82a88d429516318dcd05-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a36a6f46a23b82a88d429516318dcd05 columnFamilyName A 2024-11-23T14:25:36,144 DEBUG [StoreOpener-a36a6f46a23b82a88d429516318dcd05-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:36,144 INFO [StoreOpener-a36a6f46a23b82a88d429516318dcd05-1 {}] regionserver.HStore(327): Store=a36a6f46a23b82a88d429516318dcd05/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T14:25:36,144 INFO [StoreOpener-a36a6f46a23b82a88d429516318dcd05-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region a36a6f46a23b82a88d429516318dcd05 2024-11-23T14:25:36,146 INFO [StoreOpener-a36a6f46a23b82a88d429516318dcd05-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a36a6f46a23b82a88d429516318dcd05 columnFamilyName B 2024-11-23T14:25:36,146 DEBUG [StoreOpener-a36a6f46a23b82a88d429516318dcd05-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:36,147 INFO [StoreOpener-a36a6f46a23b82a88d429516318dcd05-1 {}] regionserver.HStore(327): Store=a36a6f46a23b82a88d429516318dcd05/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T14:25:36,147 INFO [StoreOpener-a36a6f46a23b82a88d429516318dcd05-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region a36a6f46a23b82a88d429516318dcd05 2024-11-23T14:25:36,148 INFO [StoreOpener-a36a6f46a23b82a88d429516318dcd05-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a36a6f46a23b82a88d429516318dcd05 columnFamilyName C 2024-11-23T14:25:36,148 DEBUG [StoreOpener-a36a6f46a23b82a88d429516318dcd05-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:36,149 INFO [StoreOpener-a36a6f46a23b82a88d429516318dcd05-1 {}] regionserver.HStore(327): Store=a36a6f46a23b82a88d429516318dcd05/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T14:25:36,149 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1038): replaying wal for a36a6f46a23b82a88d429516318dcd05 2024-11-23T14:25:36,149 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05 2024-11-23T14:25:36,150 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05 2024-11-23T14:25:36,151 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1048): stopping wal replay for a36a6f46a23b82a88d429516318dcd05 2024-11-23T14:25:36,151 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1060): Cleaning up temporary data for a36a6f46a23b82a88d429516318dcd05 2024-11-23T14:25:36,151 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-23T14:25:36,152 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1093): writing seq id for a36a6f46a23b82a88d429516318dcd05 2024-11-23T14:25:36,154 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T14:25:36,155 INFO [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1114): Opened a36a6f46a23b82a88d429516318dcd05; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70227929, jitterRate=0.04647769033908844}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-23T14:25:36,155 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a36a6f46a23b82a88d429516318dcd05 2024-11-23T14:25:36,156 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1006): Region open journal for a36a6f46a23b82a88d429516318dcd05: Running coprocessor pre-open hook at 1732371936140Writing region info on filesystem at 1732371936140Initializing all the Stores at 1732371936140Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732371936140Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732371936141 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732371936141Cleaning up temporary data from old regions at 1732371936151 (+10 ms)Running coprocessor post-open hooks at 1732371936155 (+4 ms)Region opened successfully at 1732371936156 (+1 ms) 2024-11-23T14:25:36,157 INFO [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., pid=38, masterSystemTime=1732371936136 2024-11-23T14:25:36,159 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:36,159 INFO [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:36,159 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=a36a6f46a23b82a88d429516318dcd05, regionState=OPEN, openSeqNum=2, regionLocation=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:36,161 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 because future has completed 2024-11-23T14:25:36,164 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=37 2024-11-23T14:25:36,164 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=37, state=SUCCESS, hasLock=false; OpenRegionProcedure a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 in 179 msec 2024-11-23T14:25:36,165 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-23T14:25:36,165 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a36a6f46a23b82a88d429516318dcd05, ASSIGN in 339 msec 2024-11-23T14:25:36,166 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-23T14:25:36,166 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732371936166"}]},"ts":"1732371936166"} 2024-11-23T14:25:36,168 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-23T14:25:36,169 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-23T14:25:36,170 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1720 sec 2024-11-23T14:25:37,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-23T14:25:37,152 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-23T14:25:37,155 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@110f9666, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:25:37,155 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6f7b1897123,38947,-1 for getting cluster id 2024-11-23T14:25:37,156 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T14:25:37,158 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c88da961-458f-4375-b9d3-275f7f90019e' 2024-11-23T14:25:37,159 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T14:25:37,159 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c88da961-458f-4375-b9d3-275f7f90019e" 2024-11-23T14:25:37,160 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23f51634, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:25:37,160 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6f7b1897123,38947,-1] 2024-11-23T14:25:37,160 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T14:25:37,161 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:25:37,163 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53032, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T14:25:37,165 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cf8b3a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:25:37,165 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T14:25:37,166 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6f7b1897123,35385,1732371902767, seqNum=-1] 2024-11-23T14:25:37,167 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T14:25:37,169 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40418, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T14:25:37,171 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-23T14:25:37,172 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is c6f7b1897123,38947,1732371901705 2024-11-23T14:25:37,172 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@ad97762 2024-11-23T14:25:37,172 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-23T14:25:37,173 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53042, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-23T14:25:37,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-23T14:25:37,180 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-23T14:25:37,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] procedure2.ProcedureExecutor(1139): Stored pid=39, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-23T14:25:37,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741971_1147 (size=995) 2024-11-23T14:25:37,600 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-23T14:25:37,600 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-23T14:25:37,603 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-23T14:25:37,609 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a36a6f46a23b82a88d429516318dcd05, REOPEN/MOVE}] 2024-11-23T14:25:37,610 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=41, ppid=40, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a36a6f46a23b82a88d429516318dcd05, REOPEN/MOVE 2024-11-23T14:25:37,611 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=41 updating hbase:meta row=a36a6f46a23b82a88d429516318dcd05, regionState=CLOSING, regionLocation=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:37,612 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=40, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a36a6f46a23b82a88d429516318dcd05, REOPEN/MOVE because future has completed 2024-11-23T14:25:37,613 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-23T14:25:37,613 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=42, ppid=41, state=RUNNABLE, hasLock=false; CloseRegionProcedure a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767}] 2024-11-23T14:25:37,766 INFO [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] handler.UnassignRegionHandler(122): Close a36a6f46a23b82a88d429516318dcd05 2024-11-23T14:25:37,766 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-23T14:25:37,766 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegion(1722): Closing a36a6f46a23b82a88d429516318dcd05, disabling compactions & flushes 2024-11-23T14:25:37,766 INFO [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:37,766 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:37,766 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. after waiting 0 ms 2024-11-23T14:25:37,766 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:37,770 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-23T14:25:37,770 INFO [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:37,771 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegion(1676): Region close journal for a36a6f46a23b82a88d429516318dcd05: Waiting for close lock at 1732371937766Running coprocessor pre-close hooks at 1732371937766Disabling compacts and flushes for region at 1732371937766Disabling writes for close at 1732371937766Writing region close event to WAL at 1732371937767 (+1 ms)Running coprocessor post-close hooks at 1732371937770 (+3 ms)Closed at 1732371937770 2024-11-23T14:25:37,771 WARN [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegionServer(3299): Not adding moved region record: a36a6f46a23b82a88d429516318dcd05 to self. 2024-11-23T14:25:37,773 INFO [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] handler.UnassignRegionHandler(157): Closed a36a6f46a23b82a88d429516318dcd05 2024-11-23T14:25:37,774 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=41 updating hbase:meta row=a36a6f46a23b82a88d429516318dcd05, regionState=CLOSED 2024-11-23T14:25:37,776 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=42, ppid=41, state=RUNNABLE, hasLock=false; CloseRegionProcedure a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 because future has completed 2024-11-23T14:25:37,779 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=42, resume processing ppid=41 2024-11-23T14:25:37,779 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, ppid=41, state=SUCCESS, hasLock=false; CloseRegionProcedure a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 in 164 msec 2024-11-23T14:25:37,780 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=41, ppid=40, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a36a6f46a23b82a88d429516318dcd05, REOPEN/MOVE; state=CLOSED, location=c6f7b1897123,35385,1732371902767; forceNewPlan=false, retain=true 2024-11-23T14:25:37,931 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=41 updating hbase:meta row=a36a6f46a23b82a88d429516318dcd05, regionState=OPENING, regionLocation=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:37,932 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=40, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a36a6f46a23b82a88d429516318dcd05, REOPEN/MOVE because future has completed 2024-11-23T14:25:37,933 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=41, state=RUNNABLE, hasLock=false; OpenRegionProcedure a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767}] 2024-11-23T14:25:38,088 INFO [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:38,088 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7752): Opening region: {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} 2024-11-23T14:25:38,088 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees a36a6f46a23b82a88d429516318dcd05 2024-11-23T14:25:38,088 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T14:25:38,089 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7794): checking encryption for a36a6f46a23b82a88d429516318dcd05 2024-11-23T14:25:38,089 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7797): checking classloading for a36a6f46a23b82a88d429516318dcd05 2024-11-23T14:25:38,092 INFO [StoreOpener-a36a6f46a23b82a88d429516318dcd05-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region a36a6f46a23b82a88d429516318dcd05 2024-11-23T14:25:38,099 INFO [StoreOpener-a36a6f46a23b82a88d429516318dcd05-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a36a6f46a23b82a88d429516318dcd05 columnFamilyName A 2024-11-23T14:25:38,101 DEBUG [StoreOpener-a36a6f46a23b82a88d429516318dcd05-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:38,101 INFO [StoreOpener-a36a6f46a23b82a88d429516318dcd05-1 {}] regionserver.HStore(327): Store=a36a6f46a23b82a88d429516318dcd05/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T14:25:38,102 INFO [StoreOpener-a36a6f46a23b82a88d429516318dcd05-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region a36a6f46a23b82a88d429516318dcd05 2024-11-23T14:25:38,103 INFO [StoreOpener-a36a6f46a23b82a88d429516318dcd05-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a36a6f46a23b82a88d429516318dcd05 columnFamilyName B 2024-11-23T14:25:38,103 DEBUG [StoreOpener-a36a6f46a23b82a88d429516318dcd05-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:38,103 INFO [StoreOpener-a36a6f46a23b82a88d429516318dcd05-1 {}] regionserver.HStore(327): Store=a36a6f46a23b82a88d429516318dcd05/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T14:25:38,103 INFO [StoreOpener-a36a6f46a23b82a88d429516318dcd05-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region a36a6f46a23b82a88d429516318dcd05 2024-11-23T14:25:38,104 INFO [StoreOpener-a36a6f46a23b82a88d429516318dcd05-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a36a6f46a23b82a88d429516318dcd05 columnFamilyName C 2024-11-23T14:25:38,104 DEBUG [StoreOpener-a36a6f46a23b82a88d429516318dcd05-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:38,104 INFO [StoreOpener-a36a6f46a23b82a88d429516318dcd05-1 {}] regionserver.HStore(327): Store=a36a6f46a23b82a88d429516318dcd05/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T14:25:38,104 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1038): replaying wal for a36a6f46a23b82a88d429516318dcd05 2024-11-23T14:25:38,105 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05 2024-11-23T14:25:38,106 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05 2024-11-23T14:25:38,106 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1048): stopping wal replay for a36a6f46a23b82a88d429516318dcd05 2024-11-23T14:25:38,106 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1060): Cleaning up temporary data for a36a6f46a23b82a88d429516318dcd05 2024-11-23T14:25:38,107 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-23T14:25:38,108 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1093): writing seq id for a36a6f46a23b82a88d429516318dcd05 2024-11-23T14:25:38,108 INFO [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1114): Opened a36a6f46a23b82a88d429516318dcd05; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69573589, jitterRate=0.036727264523506165}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-23T14:25:38,108 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a36a6f46a23b82a88d429516318dcd05 2024-11-23T14:25:38,109 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1006): Region open journal for a36a6f46a23b82a88d429516318dcd05: Running coprocessor pre-open hook at 1732371938089Writing region info on filesystem at 1732371938089Initializing all the Stores at 1732371938090 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732371938090Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732371938090Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732371938090Cleaning up temporary data from old regions at 1732371938106 (+16 ms)Running coprocessor post-open hooks at 1732371938108 (+2 ms)Region opened successfully at 1732371938109 (+1 ms) 2024-11-23T14:25:38,110 INFO [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., pid=43, masterSystemTime=1732371938085 2024-11-23T14:25:38,112 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:38,112 INFO [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:38,112 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=41 updating hbase:meta row=a36a6f46a23b82a88d429516318dcd05, regionState=OPEN, openSeqNum=5, regionLocation=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:38,114 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=43, ppid=41, state=RUNNABLE, hasLock=false; OpenRegionProcedure a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 because future has completed 2024-11-23T14:25:38,117 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=41 2024-11-23T14:25:38,117 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=41, state=SUCCESS, hasLock=false; OpenRegionProcedure a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 in 182 msec 2024-11-23T14:25:38,119 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-23T14:25:38,119 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a36a6f46a23b82a88d429516318dcd05, REOPEN/MOVE in 508 msec 2024-11-23T14:25:38,122 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=39 2024-11-23T14:25:38,122 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=39, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 517 msec 2024-11-23T14:25:38,126 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 940 msec 2024-11-23T14:25:38,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-11-23T14:25:38,127 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-23T14:25:38,128 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e0e4267, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:25:38,128 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6f7b1897123,38947,-1 for getting cluster id 2024-11-23T14:25:38,128 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T14:25:38,129 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c88da961-458f-4375-b9d3-275f7f90019e' 2024-11-23T14:25:38,129 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T14:25:38,129 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c88da961-458f-4375-b9d3-275f7f90019e" 2024-11-23T14:25:38,130 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a47cec1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:25:38,130 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6f7b1897123,38947,-1] 2024-11-23T14:25:38,130 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T14:25:38,130 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:25:38,131 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53054, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T14:25:38,132 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f0451e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:25:38,132 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4dc23ca6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:25:38,132 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6f7b1897123,38947,-1 for getting cluster id 2024-11-23T14:25:38,133 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T14:25:38,134 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c88da961-458f-4375-b9d3-275f7f90019e' 2024-11-23T14:25:38,134 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T14:25:38,134 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c88da961-458f-4375-b9d3-275f7f90019e" 2024-11-23T14:25:38,134 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17322130, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:25:38,134 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6f7b1897123,38947,-1] 2024-11-23T14:25:38,134 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T14:25:38,135 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:25:38,135 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53062, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T14:25:38,136 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27fdcd49, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:25:38,137 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@636d090d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:25:38,137 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6f7b1897123,38947,-1 for getting cluster id 2024-11-23T14:25:38,137 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T14:25:38,137 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c88da961-458f-4375-b9d3-275f7f90019e' 2024-11-23T14:25:38,138 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T14:25:38,138 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c88da961-458f-4375-b9d3-275f7f90019e" 2024-11-23T14:25:38,138 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14847f79, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:25:38,138 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6f7b1897123,38947,-1] 2024-11-23T14:25:38,138 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T14:25:38,138 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:25:38,139 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53086, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T14:25:38,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44305ffb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:25:38,140 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63d8f52d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:25:38,140 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6f7b1897123,38947,-1 for getting cluster id 2024-11-23T14:25:38,140 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T14:25:38,141 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c88da961-458f-4375-b9d3-275f7f90019e' 2024-11-23T14:25:38,141 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T14:25:38,141 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c88da961-458f-4375-b9d3-275f7f90019e" 2024-11-23T14:25:38,141 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@421816b2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:25:38,141 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6f7b1897123,38947,-1] 2024-11-23T14:25:38,142 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T14:25:38,142 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:25:38,143 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53114, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T14:25:38,143 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1049d8ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:25:38,144 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b0d456e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:25:38,144 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6f7b1897123,38947,-1 for getting cluster id 2024-11-23T14:25:38,144 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T14:25:38,145 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c88da961-458f-4375-b9d3-275f7f90019e' 2024-11-23T14:25:38,145 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T14:25:38,145 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c88da961-458f-4375-b9d3-275f7f90019e" 2024-11-23T14:25:38,145 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fb29d43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:25:38,145 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6f7b1897123,38947,-1] 2024-11-23T14:25:38,145 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T14:25:38,145 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:25:38,146 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53120, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T14:25:38,147 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28a039fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:25:38,147 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27c26b5d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:25:38,147 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6f7b1897123,38947,-1 for getting cluster id 2024-11-23T14:25:38,147 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T14:25:38,148 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c88da961-458f-4375-b9d3-275f7f90019e' 2024-11-23T14:25:38,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T14:25:38,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c88da961-458f-4375-b9d3-275f7f90019e" 2024-11-23T14:25:38,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25acc667, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:25:38,149 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6f7b1897123,38947,-1] 2024-11-23T14:25:38,149 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T14:25:38,149 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:25:38,149 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53134, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T14:25:38,150 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a0b533, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:25:38,151 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30f10c2d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:25:38,151 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6f7b1897123,38947,-1 for getting cluster id 2024-11-23T14:25:38,151 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T14:25:38,152 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c88da961-458f-4375-b9d3-275f7f90019e' 2024-11-23T14:25:38,152 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T14:25:38,152 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c88da961-458f-4375-b9d3-275f7f90019e" 2024-11-23T14:25:38,152 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a03a471, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:25:38,152 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6f7b1897123,38947,-1] 2024-11-23T14:25:38,153 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T14:25:38,153 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:25:38,153 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53164, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T14:25:38,154 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46bfc8e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:25:38,155 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7adec721, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:25:38,155 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6f7b1897123,38947,-1 for getting cluster id 2024-11-23T14:25:38,155 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T14:25:38,156 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c88da961-458f-4375-b9d3-275f7f90019e' 2024-11-23T14:25:38,156 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T14:25:38,156 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c88da961-458f-4375-b9d3-275f7f90019e" 2024-11-23T14:25:38,156 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67ba5a5f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:25:38,156 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6f7b1897123,38947,-1] 2024-11-23T14:25:38,157 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T14:25:38,157 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:25:38,158 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53190, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T14:25:38,158 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@679bff6f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:25:38,159 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78c2fc25, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:25:38,159 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6f7b1897123,38947,-1 for getting cluster id 2024-11-23T14:25:38,159 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T14:25:38,160 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c88da961-458f-4375-b9d3-275f7f90019e' 2024-11-23T14:25:38,160 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T14:25:38,160 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c88da961-458f-4375-b9d3-275f7f90019e" 2024-11-23T14:25:38,160 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@671a8f5c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:25:38,160 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6f7b1897123,38947,-1] 2024-11-23T14:25:38,160 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T14:25:38,161 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:25:38,161 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53206, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T14:25:38,162 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ac20d15, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:25:38,163 DEBUG [Thread-730 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T14:25:38,163 DEBUG [Thread-719 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T14:25:38,163 DEBUG [Thread-723 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T14:25:38,163 DEBUG [Thread-734 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T14:25:38,163 DEBUG [Thread-736 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T14:25:38,164 DEBUG [Thread-721 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T14:25:38,164 DEBUG [Thread-732 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T14:25:38,164 DEBUG [Thread-727 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T14:25:38,164 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6f7b1897123,35385,1732371902767, seqNum=-1] 2024-11-23T14:25:38,164 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6f7b1897123,35385,1732371902767, seqNum=-1] 2024-11-23T14:25:38,164 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6f7b1897123,35385,1732371902767, seqNum=-1] 2024-11-23T14:25:38,164 DEBUG [Thread-725 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T14:25:38,165 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T14:25:38,165 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T14:25:38,166 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6f7b1897123,35385,1732371902767, seqNum=-1] 2024-11-23T14:25:38,166 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6f7b1897123,35385,1732371902767, seqNum=-1] 2024-11-23T14:25:38,166 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40424, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T14:25:38,166 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6f7b1897123,35385,1732371902767, seqNum=-1] 2024-11-23T14:25:38,166 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6f7b1897123,35385,1732371902767, seqNum=-1] 2024-11-23T14:25:38,166 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T14:25:38,166 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6f7b1897123,35385,1732371902767, seqNum=-1] 2024-11-23T14:25:38,166 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T14:25:38,166 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T14:25:38,166 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T14:25:38,166 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T14:25:38,166 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6f7b1897123,35385,1732371902767, seqNum=-1] 2024-11-23T14:25:38,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T14:25:38,167 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40434, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T14:25:38,168 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-23T14:25:38,168 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40436, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T14:25:38,168 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T14:25:38,168 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40462, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T14:25:38,169 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40476, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T14:25:38,169 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40438, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T14:25:38,169 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5] 2024-11-23T14:25:38,169 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40464, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T14:25:38,170 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40478, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T14:25:38,170 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5] 2024-11-23T14:25:38,171 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40492, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T14:25:38,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5] 2024-11-23T14:25:38,171 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5] 2024-11-23T14:25:38,171 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5] 2024-11-23T14:25:38,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-23T14:25:38,172 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5] 2024-11-23T14:25:38,173 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T14:25:38,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-23T14:25:38,174 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T14:25:38,174 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T14:25:38,175 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5] 2024-11-23T14:25:38,176 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5] 2024-11-23T14:25:38,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5] 2024-11-23T14:25:38,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on a36a6f46a23b82a88d429516318dcd05 2024-11-23T14:25:38,200 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T14:25:38,200 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a36a6f46a23b82a88d429516318dcd05 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-23T14:25:38,244 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123768628a7a7164cd1a12c65ea6f937c8e_a36a6f46a23b82a88d429516318dcd05 is 50, key is test_row_0/A:col10/1732371938191/Put/seqid=0 2024-11-23T14:25:38,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741972_1148 (size=17034) 2024-11-23T14:25:38,267 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:38,275 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123768628a7a7164cd1a12c65ea6f937c8e_a36a6f46a23b82a88d429516318dcd05 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123768628a7a7164cd1a12c65ea6f937c8e_a36a6f46a23b82a88d429516318dcd05 2024-11-23T14:25:38,277 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/A/8b9a4536085440c69642aa67c480a522, store: [table=TestAcidGuarantees family=A region=a36a6f46a23b82a88d429516318dcd05] 2024-11-23T14:25:38,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:38,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40438 deadline: 1732371998274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:38,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-23T14:25:38,282 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:38,282 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:38,282 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:38,285 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:38,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40436 deadline: 1732371998277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:38,286 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:38,286 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:38,286 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:38,286 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:38,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40478 deadline: 1732371998278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:38,287 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:38,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:38,287 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:38,287 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:38,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40464 deadline: 1732371998282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:38,288 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:38,288 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:38,288 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:38,293 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/A/8b9a4536085440c69642aa67c480a522 is 175, key is test_row_0/A:col10/1732371938191/Put/seqid=0 2024-11-23T14:25:38,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:38,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40476 deadline: 1732371998288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:38,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:38,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:38,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:38,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741973_1149 (size=48139) 2024-11-23T14:25:38,309 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/A/8b9a4536085440c69642aa67c480a522 2024-11-23T14:25:38,335 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-23T14:25:38,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:38,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. as already flushing 2024-11-23T14:25:38,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:38,335 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:38,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:38,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:38,362 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/B/660b0205af5d41549c89e222c938de2e is 50, key is test_row_0/B:col10/1732371938191/Put/seqid=0 2024-11-23T14:25:38,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741974_1150 (size=12001) 2024-11-23T14:25:38,381 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/B/660b0205af5d41549c89e222c938de2e 2024-11-23T14:25:38,392 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:38,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40438 deadline: 1732371998390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:38,393 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:38,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40436 deadline: 1732371998390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:38,394 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:38,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:38,394 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:38,394 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:38,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40478 deadline: 1732371998391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:38,394 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:38,394 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:38,394 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:38,394 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:38,394 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:38,394 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:38,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:38,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40464 deadline: 1732371998391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:38,395 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:38,395 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:38,395 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:38,405 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:38,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40476 deadline: 1732371998400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:38,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:38,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:38,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:38,425 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/C/5b26b12dff5d470a9f34ed0d875702ea is 50, key is test_row_0/C:col10/1732371938191/Put/seqid=0 2024-11-23T14:25:38,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741975_1151 (size=12001) 2024-11-23T14:25:38,452 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/C/5b26b12dff5d470a9f34ed0d875702ea 2024-11-23T14:25:38,462 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/A/8b9a4536085440c69642aa67c480a522 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/8b9a4536085440c69642aa67c480a522 2024-11-23T14:25:38,472 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/8b9a4536085440c69642aa67c480a522, entries=250, sequenceid=16, filesize=47.0 K 2024-11-23T14:25:38,475 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/B/660b0205af5d41549c89e222c938de2e as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/660b0205af5d41549c89e222c938de2e 2024-11-23T14:25:38,487 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/660b0205af5d41549c89e222c938de2e, entries=150, sequenceid=16, filesize=11.7 K 2024-11-23T14:25:38,488 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-23T14:25:38,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:38,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. as already flushing 2024-11-23T14:25:38,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:38,489 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:38,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:38,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:38,490 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/C/5b26b12dff5d470a9f34ed0d875702ea as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/5b26b12dff5d470a9f34ed0d875702ea 2024-11-23T14:25:38,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-23T14:25:38,495 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/5b26b12dff5d470a9f34ed0d875702ea, entries=150, sequenceid=16, filesize=11.7 K 2024-11-23T14:25:38,496 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for a36a6f46a23b82a88d429516318dcd05 in 296ms, sequenceid=16, compaction requested=false 2024-11-23T14:25:38,497 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-23T14:25:38,498 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a36a6f46a23b82a88d429516318dcd05: 2024-11-23T14:25:38,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on a36a6f46a23b82a88d429516318dcd05 2024-11-23T14:25:38,603 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T14:25:38,603 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a36a6f46a23b82a88d429516318dcd05 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-23T14:25:38,617 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123192e55b1f3dd4b7299f2eaef1ee0f59c_a36a6f46a23b82a88d429516318dcd05 is 50, key is test_row_0/A:col10/1732371938602/Put/seqid=0 2024-11-23T14:25:38,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:38,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40476 deadline: 1732371998611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:38,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:38,618 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:38,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:38,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:38,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40464 deadline: 1732371998611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:38,619 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:38,619 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:38,619 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:38,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:38,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40436 deadline: 1732371998613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:38,620 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:38,620 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:38,620 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:38,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:38,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40438 deadline: 1732371998618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:38,622 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:38,622 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:38,622 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:38,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:38,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40478 deadline: 1732371998620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:38,626 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:38,626 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:38,626 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:38,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741976_1152 (size=12154) 2024-11-23T14:25:38,633 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:38,642 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-23T14:25:38,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:38,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. as already flushing 2024-11-23T14:25:38,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:38,643 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:38,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:38,644 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123192e55b1f3dd4b7299f2eaef1ee0f59c_a36a6f46a23b82a88d429516318dcd05 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123192e55b1f3dd4b7299f2eaef1ee0f59c_a36a6f46a23b82a88d429516318dcd05 2024-11-23T14:25:38,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:38,646 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/A/7f02d7fa060445c49a8c1113212bdbc1, store: [table=TestAcidGuarantees family=A region=a36a6f46a23b82a88d429516318dcd05] 2024-11-23T14:25:38,647 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/A/7f02d7fa060445c49a8c1113212bdbc1 is 175, key is test_row_0/A:col10/1732371938602/Put/seqid=0 2024-11-23T14:25:38,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741977_1153 (size=30955) 2024-11-23T14:25:38,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:38,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40464 deadline: 1732371998730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:38,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:38,732 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:38,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40438 deadline: 1732371998730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:38,732 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:38,732 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:38,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:38,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40436 deadline: 1732371998731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:38,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:38,733 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:38,733 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:38,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40478 deadline: 1732371998731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:38,733 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:38,733 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:38,733 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:38,733 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:38,733 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:38,733 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:38,734 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:38,797 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-23T14:25:38,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:38,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. as already flushing 2024-11-23T14:25:38,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:38,798 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:38,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:38,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:38,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-23T14:25:38,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:38,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40476 deadline: 1732371998931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:38,932 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:38,933 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:38,933 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:38,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:38,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40464 deadline: 1732371998940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:38,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:38,943 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:38,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40436 deadline: 1732371998940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:38,943 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:38,943 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:38,943 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:38,943 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:38,943 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:38,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40438 deadline: 1732371998940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:38,943 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:38,943 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:38,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40478 deadline: 1732371998941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:38,944 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:38,944 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:38,944 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:38,945 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:38,945 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:38,945 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:38,954 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-23T14:25:38,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:38,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. as already flushing 2024-11-23T14:25:38,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:38,954 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:38,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:38,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:39,095 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=44, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/A/7f02d7fa060445c49a8c1113212bdbc1 2024-11-23T14:25:39,109 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-23T14:25:39,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:39,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. as already flushing 2024-11-23T14:25:39,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:39,110 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:39,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:39,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:39,203 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/B/2ae303836ea34be9afc87a30c9eb7a13 is 50, key is test_row_0/B:col10/1732371938602/Put/seqid=0 2024-11-23T14:25:39,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741978_1154 (size=12001) 2024-11-23T14:25:39,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:39,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40478 deadline: 1732371999251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:39,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:39,253 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:39,253 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:39,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40436 deadline: 1732371999251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:39,253 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:39,253 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:39,253 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:39,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:39,253 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:39,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40464 deadline: 1732371999251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:39,253 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:39,254 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:39,254 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:39,254 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:39,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40438 deadline: 1732371999252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:39,255 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:39,255 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:39,255 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:39,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-23T14:25:39,347 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-23T14:25:39,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:39,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. as already flushing 2024-11-23T14:25:39,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:39,348 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:39,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:39,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:39,446 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:39,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40476 deadline: 1732371999441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:39,447 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:39,448 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:39,448 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:39,502 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-23T14:25:39,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:39,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. as already flushing 2024-11-23T14:25:39,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:39,502 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:39,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:39,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:39,580 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-23T14:25:39,634 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=44 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/B/2ae303836ea34be9afc87a30c9eb7a13 2024-11-23T14:25:39,644 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/C/1692f31d1986489f80bfb1803959d95c is 50, key is test_row_0/C:col10/1732371938602/Put/seqid=0 2024-11-23T14:25:39,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741979_1155 (size=12001) 2024-11-23T14:25:39,655 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-23T14:25:39,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:39,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. as already flushing 2024-11-23T14:25:39,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:39,655 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:39,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:39,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:39,761 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:39,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40436 deadline: 1732371999760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:39,761 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:39,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40478 deadline: 1732371999761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:39,761 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:39,762 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:39,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40464 deadline: 1732371999761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:39,762 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:39,762 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:39,762 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:39,762 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:39,762 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:39,762 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:39,762 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:39,762 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:39,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:39,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40438 deadline: 1732371999770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:39,771 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:39,772 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:39,772 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:39,808 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-23T14:25:39,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:39,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. as already flushing 2024-11-23T14:25:39,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:39,808 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:39,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:39,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:39,963 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-23T14:25:39,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:39,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. as already flushing 2024-11-23T14:25:39,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:39,963 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:39,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:39,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:40,049 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=44 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/C/1692f31d1986489f80bfb1803959d95c 2024-11-23T14:25:40,054 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/A/7f02d7fa060445c49a8c1113212bdbc1 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/7f02d7fa060445c49a8c1113212bdbc1 2024-11-23T14:25:40,060 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/7f02d7fa060445c49a8c1113212bdbc1, entries=150, sequenceid=44, filesize=30.2 K 2024-11-23T14:25:40,062 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/B/2ae303836ea34be9afc87a30c9eb7a13 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/2ae303836ea34be9afc87a30c9eb7a13 2024-11-23T14:25:40,068 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/2ae303836ea34be9afc87a30c9eb7a13, entries=150, sequenceid=44, filesize=11.7 K 2024-11-23T14:25:40,069 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/C/1692f31d1986489f80bfb1803959d95c as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/1692f31d1986489f80bfb1803959d95c 2024-11-23T14:25:40,087 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/1692f31d1986489f80bfb1803959d95c, entries=150, sequenceid=44, filesize=11.7 K 2024-11-23T14:25:40,089 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=40.25 KB/41220 for a36a6f46a23b82a88d429516318dcd05 in 1486ms, sequenceid=44, compaction requested=false 2024-11-23T14:25:40,089 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a36a6f46a23b82a88d429516318dcd05: 2024-11-23T14:25:40,117 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-23T14:25:40,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:40,117 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing a36a6f46a23b82a88d429516318dcd05 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-23T14:25:40,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112319992e64b7e94b2db995c33d49bcbc68_a36a6f46a23b82a88d429516318dcd05 is 50, key is test_row_0/A:col10/1732371938612/Put/seqid=0 2024-11-23T14:25:40,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741980_1156 (size=12154) 2024-11-23T14:25:40,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:40,150 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112319992e64b7e94b2db995c33d49bcbc68_a36a6f46a23b82a88d429516318dcd05 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112319992e64b7e94b2db995c33d49bcbc68_a36a6f46a23b82a88d429516318dcd05 2024-11-23T14:25:40,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/A/86ebc0a873dc4a4483a5733acde72e81, store: [table=TestAcidGuarantees family=A region=a36a6f46a23b82a88d429516318dcd05] 2024-11-23T14:25:40,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/A/86ebc0a873dc4a4483a5733acde72e81 is 175, key is test_row_0/A:col10/1732371938612/Put/seqid=0 2024-11-23T14:25:40,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741981_1157 (size=30955) 2024-11-23T14:25:40,166 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=53, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/A/86ebc0a873dc4a4483a5733acde72e81 2024-11-23T14:25:40,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/B/f964cc4621f140febad7b00ccd4bc475 is 50, key is test_row_0/B:col10/1732371938612/Put/seqid=0 2024-11-23T14:25:40,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741982_1158 (size=12001) 2024-11-23T14:25:40,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-23T14:25:40,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on a36a6f46a23b82a88d429516318dcd05 2024-11-23T14:25:40,467 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. as already flushing 2024-11-23T14:25:40,572 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:40,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40476 deadline: 1732372000569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:40,573 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:40,573 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:40,573 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:40,593 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/B/f964cc4621f140febad7b00ccd4bc475 2024-11-23T14:25:40,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/C/ed60c40b23b64a94b075891ae055813d is 50, key is test_row_0/C:col10/1732371938612/Put/seqid=0 2024-11-23T14:25:40,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741983_1159 (size=12001) 2024-11-23T14:25:40,618 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/C/ed60c40b23b64a94b075891ae055813d 2024-11-23T14:25:40,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/A/86ebc0a873dc4a4483a5733acde72e81 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/86ebc0a873dc4a4483a5733acde72e81 2024-11-23T14:25:40,639 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/86ebc0a873dc4a4483a5733acde72e81, entries=150, sequenceid=53, filesize=30.2 K 2024-11-23T14:25:40,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/B/f964cc4621f140febad7b00ccd4bc475 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/f964cc4621f140febad7b00ccd4bc475 2024-11-23T14:25:40,651 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/f964cc4621f140febad7b00ccd4bc475, entries=150, sequenceid=53, filesize=11.7 K 2024-11-23T14:25:40,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/C/ed60c40b23b64a94b075891ae055813d as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/ed60c40b23b64a94b075891ae055813d 2024-11-23T14:25:40,661 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/ed60c40b23b64a94b075891ae055813d, entries=150, sequenceid=53, filesize=11.7 K 2024-11-23T14:25:40,662 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=161.02 KB/164880 for a36a6f46a23b82a88d429516318dcd05 in 545ms, sequenceid=53, compaction requested=true 2024-11-23T14:25:40,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for a36a6f46a23b82a88d429516318dcd05: 2024-11-23T14:25:40,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:40,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-23T14:25:40,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-23T14:25:40,668 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-23T14:25:40,668 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4920 sec 2024-11-23T14:25:40,673 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 2.5010 sec 2024-11-23T14:25:40,685 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T14:25:40,686 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a36a6f46a23b82a88d429516318dcd05 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-23T14:25:40,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on a36a6f46a23b82a88d429516318dcd05 2024-11-23T14:25:40,707 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112304b9d2dec44e4c4a8fef37b3b557b124_a36a6f46a23b82a88d429516318dcd05 is 50, key is test_row_0/A:col10/1732371940567/Put/seqid=0 2024-11-23T14:25:40,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:40,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40476 deadline: 1732372000717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:40,718 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:40,718 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:40,718 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:40,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741984_1160 (size=14594) 2024-11-23T14:25:40,728 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:40,735 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112304b9d2dec44e4c4a8fef37b3b557b124_a36a6f46a23b82a88d429516318dcd05 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112304b9d2dec44e4c4a8fef37b3b557b124_a36a6f46a23b82a88d429516318dcd05 2024-11-23T14:25:40,737 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/A/80762520f40c464495264844b141f777, store: [table=TestAcidGuarantees family=A region=a36a6f46a23b82a88d429516318dcd05] 2024-11-23T14:25:40,738 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/A/80762520f40c464495264844b141f777 is 175, key is test_row_0/A:col10/1732371940567/Put/seqid=0 2024-11-23T14:25:40,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741985_1161 (size=39549) 2024-11-23T14:25:40,769 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=81, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/A/80762520f40c464495264844b141f777 2024-11-23T14:25:40,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:40,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40464 deadline: 1732372000770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:40,772 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to c6f7b1897123,35385,1732371902767 for 'test_row_2' in a36a6f46a23b82a88d429516318dcd05 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-23T14:25:40,772 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:40,772 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:40,772 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:40,773 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:40,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40436 deadline: 1732372000772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:40,774 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to c6f7b1897123,35385,1732371902767 for 'test_row_0' in a36a6f46a23b82a88d429516318dcd05 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-23T14:25:40,774 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:40,774 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:40,774 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:40,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:40,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40478 deadline: 1732372000780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:40,786 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to c6f7b1897123,35385,1732371902767 for 'test_row_0' in a36a6f46a23b82a88d429516318dcd05 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-23T14:25:40,786 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:40,786 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:40,786 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:40,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:40,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40438 deadline: 1732372000790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:40,795 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to c6f7b1897123,35385,1732371902767 for 'test_row_0' in a36a6f46a23b82a88d429516318dcd05 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-23T14:25:40,795 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:40,796 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:40,796 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:40,807 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/B/b606e7ced1994665a65e30549da2b449 is 50, key is test_row_0/B:col10/1732371940567/Put/seqid=0 2024-11-23T14:25:40,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741986_1162 (size=12001) 2024-11-23T14:25:40,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:40,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40476 deadline: 1732372000830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:40,833 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:40,834 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:40,834 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:41,051 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:41,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40476 deadline: 1732372001049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:41,052 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:41,052 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:41,052 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:41,229 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/B/b606e7ced1994665a65e30549da2b449 2024-11-23T14:25:41,253 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/C/1f49586a29fe48679d909ceeb0805b5d is 50, key is test_row_0/C:col10/1732371940567/Put/seqid=0 2024-11-23T14:25:41,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741987_1163 (size=12001) 2024-11-23T14:25:41,263 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/C/1f49586a29fe48679d909ceeb0805b5d 2024-11-23T14:25:41,271 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/A/80762520f40c464495264844b141f777 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/80762520f40c464495264844b141f777 2024-11-23T14:25:41,276 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/80762520f40c464495264844b141f777, entries=200, sequenceid=81, filesize=38.6 K 2024-11-23T14:25:41,280 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/B/b606e7ced1994665a65e30549da2b449 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/b606e7ced1994665a65e30549da2b449 2024-11-23T14:25:41,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,296 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/b606e7ced1994665a65e30549da2b449, entries=150, sequenceid=81, filesize=11.7 K 2024-11-23T14:25:41,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,298 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/C/1f49586a29fe48679d909ceeb0805b5d as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/1f49586a29fe48679d909ceeb0805b5d 2024-11-23T14:25:41,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,303 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/1f49586a29fe48679d909ceeb0805b5d, entries=150, sequenceid=81, filesize=11.7 K 2024-11-23T14:25:41,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,304 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for a36a6f46a23b82a88d429516318dcd05 in 619ms, sequenceid=81, compaction requested=true 2024-11-23T14:25:41,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,304 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a36a6f46a23b82a88d429516318dcd05: 2024-11-23T14:25:41,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a36a6f46a23b82a88d429516318dcd05:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T14:25:41,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:25:41,304 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T14:25:41,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a36a6f46a23b82a88d429516318dcd05:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T14:25:41,305 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T14:25:41,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:25:41,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a36a6f46a23b82a88d429516318dcd05:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T14:25:41,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:25:41,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,306 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 149598 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T14:25:41,306 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T14:25:41,306 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1541): a36a6f46a23b82a88d429516318dcd05/A is initiating minor compaction (all files) 2024-11-23T14:25:41,306 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1541): a36a6f46a23b82a88d429516318dcd05/B is initiating minor compaction (all files) 2024-11-23T14:25:41,306 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a36a6f46a23b82a88d429516318dcd05/A in TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:41,306 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a36a6f46a23b82a88d429516318dcd05/B in TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:41,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,306 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/8b9a4536085440c69642aa67c480a522, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/7f02d7fa060445c49a8c1113212bdbc1, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/86ebc0a873dc4a4483a5733acde72e81, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/80762520f40c464495264844b141f777] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp, totalSize=146.1 K 2024-11-23T14:25:41,306 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/660b0205af5d41549c89e222c938de2e, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/2ae303836ea34be9afc87a30c9eb7a13, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/f964cc4621f140febad7b00ccd4bc475, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/b606e7ced1994665a65e30549da2b449] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp, totalSize=46.9 K 2024-11-23T14:25:41,307 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:41,307 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. files: [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/8b9a4536085440c69642aa67c480a522, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/7f02d7fa060445c49a8c1113212bdbc1, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/86ebc0a873dc4a4483a5733acde72e81, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/80762520f40c464495264844b141f777] 2024-11-23T14:25:41,307 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8b9a4536085440c69642aa67c480a522, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732371938175 2024-11-23T14:25:41,307 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 660b0205af5d41549c89e222c938de2e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732371938191 2024-11-23T14:25:41,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,307 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f02d7fa060445c49a8c1113212bdbc1, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=44, earliestPutTs=1732371938274 2024-11-23T14:25:41,308 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ae303836ea34be9afc87a30c9eb7a13, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=44, earliestPutTs=1732371938274 2024-11-23T14:25:41,308 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 86ebc0a873dc4a4483a5733acde72e81, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732371938607 2024-11-23T14:25:41,308 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting f964cc4621f140febad7b00ccd4bc475, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732371938607 2024-11-23T14:25:41,308 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting b606e7ced1994665a65e30549da2b449, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1732371940530 2024-11-23T14:25:41,308 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 80762520f40c464495264844b141f777, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1732371940530 2024-11-23T14:25:41,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,327 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a36a6f46a23b82a88d429516318dcd05#B#compaction#144 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:25:41,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,329 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a36a6f46a23b82a88d429516318dcd05] 2024-11-23T14:25:41,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,331 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/B/e080fdca63e84552ad50f80a6ce3d44a is 50, key is test_row_0/B:col10/1732371940567/Put/seqid=0 2024-11-23T14:25:41,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,332 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411238dfa2ecc97154734afe1e6ebd18ef4d4_a36a6f46a23b82a88d429516318dcd05 store=[table=TestAcidGuarantees family=A region=a36a6f46a23b82a88d429516318dcd05] 2024-11-23T14:25:41,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,342 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411238dfa2ecc97154734afe1e6ebd18ef4d4_a36a6f46a23b82a88d429516318dcd05, store=[table=TestAcidGuarantees family=A region=a36a6f46a23b82a88d429516318dcd05] 2024-11-23T14:25:41,342 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411238dfa2ecc97154734afe1e6ebd18ef4d4_a36a6f46a23b82a88d429516318dcd05 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a36a6f46a23b82a88d429516318dcd05] 2024-11-23T14:25:41,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741988_1164 (size=12139) 2024-11-23T14:25:41,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741989_1165 (size=4469) 2024-11-23T14:25:41,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,418 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a36a6f46a23b82a88d429516318dcd05#A#compaction#145 average throughput is 0.27 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:25:41,419 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/A/a76a96cbec9b4d838c5f575d3708c9d7 is 175, key is test_row_0/A:col10/1732371940567/Put/seqid=0 2024-11-23T14:25:41,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,429 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/B/e080fdca63e84552ad50f80a6ce3d44a as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/e080fdca63e84552ad50f80a6ce3d44a 2024-11-23T14:25:41,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,436 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a36a6f46a23b82a88d429516318dcd05/B of a36a6f46a23b82a88d429516318dcd05 into e080fdca63e84552ad50f80a6ce3d44a(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:25:41,436 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a36a6f46a23b82a88d429516318dcd05: 2024-11-23T14:25:41,436 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., storeName=a36a6f46a23b82a88d429516318dcd05/B, priority=12, startTime=1732371941305; duration=0sec 2024-11-23T14:25:41,436 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:25:41,436 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a36a6f46a23b82a88d429516318dcd05:B 2024-11-23T14:25:41,436 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T14:25:41,438 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T14:25:41,438 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1541): a36a6f46a23b82a88d429516318dcd05/C is initiating minor compaction (all files) 2024-11-23T14:25:41,438 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a36a6f46a23b82a88d429516318dcd05/C in TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:41,438 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/5b26b12dff5d470a9f34ed0d875702ea, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/1692f31d1986489f80bfb1803959d95c, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/ed60c40b23b64a94b075891ae055813d, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/1f49586a29fe48679d909ceeb0805b5d] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp, totalSize=46.9 K 2024-11-23T14:25:41,439 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b26b12dff5d470a9f34ed0d875702ea, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732371938191 2024-11-23T14:25:41,440 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 1692f31d1986489f80bfb1803959d95c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=44, earliestPutTs=1732371938274 2024-11-23T14:25:41,440 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting ed60c40b23b64a94b075891ae055813d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732371938607 2024-11-23T14:25:41,441 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f49586a29fe48679d909ceeb0805b5d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1732371940530 2024-11-23T14:25:41,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741990_1166 (size=31093) 2024-11-23T14:25:41,457 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/A/a76a96cbec9b4d838c5f575d3708c9d7 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/a76a96cbec9b4d838c5f575d3708c9d7 2024-11-23T14:25:41,462 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a36a6f46a23b82a88d429516318dcd05#C#compaction#146 average throughput is 0.73 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:25:41,462 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/C/78154b6552814181a3a53575262732eb is 50, key is test_row_0/C:col10/1732371940567/Put/seqid=0 2024-11-23T14:25:41,467 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a36a6f46a23b82a88d429516318dcd05/A of a36a6f46a23b82a88d429516318dcd05 into a76a96cbec9b4d838c5f575d3708c9d7(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:25:41,467 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a36a6f46a23b82a88d429516318dcd05: 2024-11-23T14:25:41,467 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., storeName=a36a6f46a23b82a88d429516318dcd05/A, priority=12, startTime=1732371941304; duration=0sec 2024-11-23T14:25:41,467 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:25:41,467 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a36a6f46a23b82a88d429516318dcd05:A 2024-11-23T14:25:41,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on a36a6f46a23b82a88d429516318dcd05 2024-11-23T14:25:41,468 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T14:25:41,468 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a36a6f46a23b82a88d429516318dcd05 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-23T14:25:41,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741991_1167 (size=12139) 2024-11-23T14:25:41,492 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411230a4646e92d6d42d2bc2ce4b06000c2c9_a36a6f46a23b82a88d429516318dcd05 is 50, key is test_row_0/A:col10/1732371941448/Put/seqid=0 2024-11-23T14:25:41,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741992_1168 (size=17034) 2024-11-23T14:25:41,512 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:41,519 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411230a4646e92d6d42d2bc2ce4b06000c2c9_a36a6f46a23b82a88d429516318dcd05 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411230a4646e92d6d42d2bc2ce4b06000c2c9_a36a6f46a23b82a88d429516318dcd05 2024-11-23T14:25:41,520 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/A/dd868ed7718c4340a463cc0c6951ff27, store: [table=TestAcidGuarantees family=A region=a36a6f46a23b82a88d429516318dcd05] 2024-11-23T14:25:41,521 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/A/dd868ed7718c4340a463cc0c6951ff27 is 175, key is test_row_0/A:col10/1732371941448/Put/seqid=0 2024-11-23T14:25:41,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741993_1169 (size=48139) 2024-11-23T14:25:41,545 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=94, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/A/dd868ed7718c4340a463cc0c6951ff27 2024-11-23T14:25:41,556 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/B/500ec3da77a34ecbbba528b6865f204b is 50, key is test_row_0/B:col10/1732371941448/Put/seqid=0 2024-11-23T14:25:41,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741994_1170 (size=12001) 2024-11-23T14:25:41,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:41,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40476 deadline: 1732372001602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:41,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:41,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:41,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:41,711 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:41,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40476 deadline: 1732372001710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:41,712 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:41,712 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:41,712 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:41,892 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/C/78154b6552814181a3a53575262732eb as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/78154b6552814181a3a53575262732eb 2024-11-23T14:25:41,899 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a36a6f46a23b82a88d429516318dcd05/C of a36a6f46a23b82a88d429516318dcd05 into 78154b6552814181a3a53575262732eb(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:25:41,899 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a36a6f46a23b82a88d429516318dcd05: 2024-11-23T14:25:41,899 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., storeName=a36a6f46a23b82a88d429516318dcd05/C, priority=12, startTime=1732371941305; duration=0sec 2024-11-23T14:25:41,899 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:25:41,900 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a36a6f46a23b82a88d429516318dcd05:C 2024-11-23T14:25:41,922 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:41,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40476 deadline: 1732372001920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:41,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:41,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:41,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:41,983 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/B/500ec3da77a34ecbbba528b6865f204b 2024-11-23T14:25:41,992 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/C/f5070d71c7104d53bedd73258d5d022b is 50, key is test_row_0/C:col10/1732371941448/Put/seqid=0 2024-11-23T14:25:41,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741995_1171 (size=12001) 2024-11-23T14:25:42,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:42,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40476 deadline: 1732372002230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:42,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:42,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:42,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:42,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-23T14:25:42,350 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T14:25:42,350 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-23T14:25:42,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-23T14:25:42,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-23T14:25:42,352 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T14:25:42,352 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T14:25:42,352 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T14:25:42,399 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/C/f5070d71c7104d53bedd73258d5d022b 2024-11-23T14:25:42,403 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/A/dd868ed7718c4340a463cc0c6951ff27 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/dd868ed7718c4340a463cc0c6951ff27 2024-11-23T14:25:42,408 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/dd868ed7718c4340a463cc0c6951ff27, entries=250, sequenceid=94, filesize=47.0 K 2024-11-23T14:25:42,409 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/B/500ec3da77a34ecbbba528b6865f204b as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/500ec3da77a34ecbbba528b6865f204b 2024-11-23T14:25:42,413 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/500ec3da77a34ecbbba528b6865f204b, entries=150, sequenceid=94, filesize=11.7 K 2024-11-23T14:25:42,420 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/C/f5070d71c7104d53bedd73258d5d022b as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/f5070d71c7104d53bedd73258d5d022b 2024-11-23T14:25:42,429 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/f5070d71c7104d53bedd73258d5d022b, entries=150, sequenceid=94, filesize=11.7 K 2024-11-23T14:25:42,430 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for a36a6f46a23b82a88d429516318dcd05 in 962ms, sequenceid=94, compaction requested=false 2024-11-23T14:25:42,430 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a36a6f46a23b82a88d429516318dcd05: 2024-11-23T14:25:42,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-23T14:25:42,503 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-23T14:25:42,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:42,504 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing a36a6f46a23b82a88d429516318dcd05 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-23T14:25:42,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112328a81ccb546448fca1dc869eb0f27370_a36a6f46a23b82a88d429516318dcd05 is 50, key is test_row_0/A:col10/1732371941597/Put/seqid=0 2024-11-23T14:25:42,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741996_1172 (size=12154) 2024-11-23T14:25:42,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:42,524 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112328a81ccb546448fca1dc869eb0f27370_a36a6f46a23b82a88d429516318dcd05 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112328a81ccb546448fca1dc869eb0f27370_a36a6f46a23b82a88d429516318dcd05 2024-11-23T14:25:42,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/A/8f12c3e5ecbb49d49e15a181521e0a05, store: [table=TestAcidGuarantees family=A region=a36a6f46a23b82a88d429516318dcd05] 2024-11-23T14:25:42,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/A/8f12c3e5ecbb49d49e15a181521e0a05 is 175, key is test_row_0/A:col10/1732371941597/Put/seqid=0 2024-11-23T14:25:42,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741997_1173 (size=30955) 2024-11-23T14:25:42,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-23T14:25:42,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on a36a6f46a23b82a88d429516318dcd05 2024-11-23T14:25:42,742 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. as already flushing 2024-11-23T14:25:42,764 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:42,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40476 deadline: 1732372002762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:42,764 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:42,765 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:42,765 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:42,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:42,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40464 deadline: 1732372002780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:42,781 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to c6f7b1897123,35385,1732371902767 for 'test_row_2' in a36a6f46a23b82a88d429516318dcd05 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-23T14:25:42,782 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:42,782 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:42,782 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:42,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:42,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40436 deadline: 1732372002790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:42,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:42,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40478 deadline: 1732372002790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:42,792 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to c6f7b1897123,35385,1732371902767 for 'test_row_0' in a36a6f46a23b82a88d429516318dcd05 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-23T14:25:42,793 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:42,793 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:42,793 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:42,793 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to c6f7b1897123,35385,1732371902767 for 'test_row_0' in a36a6f46a23b82a88d429516318dcd05 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-23T14:25:42,793 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:42,793 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:42,793 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:42,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:42,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40438 deadline: 1732372002810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:42,816 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to c6f7b1897123,35385,1732371902767 for 'test_row_0' in a36a6f46a23b82a88d429516318dcd05 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-23T14:25:42,817 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:42,817 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:42,817 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:42,874 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:42,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40476 deadline: 1732372002870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:42,874 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:42,874 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:42,874 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:42,940 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=120, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/A/8f12c3e5ecbb49d49e15a181521e0a05 2024-11-23T14:25:42,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/B/35ac91d05fab48729159f47a56d86816 is 50, key is test_row_0/B:col10/1732371941597/Put/seqid=0 2024-11-23T14:25:42,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741998_1174 (size=12001) 2024-11-23T14:25:42,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-23T14:25:43,082 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:43,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40476 deadline: 1732372003080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:43,082 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:43,082 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:43,082 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:43,357 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/B/35ac91d05fab48729159f47a56d86816 2024-11-23T14:25:43,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/C/0f3695881b19489faba6a663bf85188f is 50, key is test_row_0/C:col10/1732371941597/Put/seqid=0 2024-11-23T14:25:43,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741999_1175 (size=12001) 2024-11-23T14:25:43,392 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:43,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40476 deadline: 1732372003390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:43,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:43,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:43,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:43,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-23T14:25:43,768 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/C/0f3695881b19489faba6a663bf85188f 2024-11-23T14:25:43,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/A/8f12c3e5ecbb49d49e15a181521e0a05 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/8f12c3e5ecbb49d49e15a181521e0a05 2024-11-23T14:25:43,777 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/8f12c3e5ecbb49d49e15a181521e0a05, entries=150, sequenceid=120, filesize=30.2 K 2024-11-23T14:25:43,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/B/35ac91d05fab48729159f47a56d86816 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/35ac91d05fab48729159f47a56d86816 2024-11-23T14:25:43,783 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/35ac91d05fab48729159f47a56d86816, entries=150, sequenceid=120, filesize=11.7 K 2024-11-23T14:25:43,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/C/0f3695881b19489faba6a663bf85188f as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/0f3695881b19489faba6a663bf85188f 2024-11-23T14:25:43,790 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/0f3695881b19489faba6a663bf85188f, entries=150, sequenceid=120, filesize=11.7 K 2024-11-23T14:25:43,791 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for a36a6f46a23b82a88d429516318dcd05 in 1286ms, sequenceid=120, compaction requested=true 2024-11-23T14:25:43,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for a36a6f46a23b82a88d429516318dcd05: 2024-11-23T14:25:43,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:43,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-23T14:25:43,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-23T14:25:43,794 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-23T14:25:43,794 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4400 sec 2024-11-23T14:25:43,796 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 1.4440 sec 2024-11-23T14:25:43,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on a36a6f46a23b82a88d429516318dcd05 2024-11-23T14:25:43,902 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T14:25:43,902 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a36a6f46a23b82a88d429516318dcd05 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-23T14:25:43,909 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411232ff5c73239a946ae975768c1e7517ac2_a36a6f46a23b82a88d429516318dcd05 is 50, key is test_row_0/A:col10/1732371942759/Put/seqid=0 2024-11-23T14:25:43,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742000_1176 (size=12254) 2024-11-23T14:25:43,919 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:43,924 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411232ff5c73239a946ae975768c1e7517ac2_a36a6f46a23b82a88d429516318dcd05 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411232ff5c73239a946ae975768c1e7517ac2_a36a6f46a23b82a88d429516318dcd05 2024-11-23T14:25:43,925 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/A/c2e10ffca30b4adfb1d07e2fc004a139, store: [table=TestAcidGuarantees family=A region=a36a6f46a23b82a88d429516318dcd05] 2024-11-23T14:25:43,926 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/A/c2e10ffca30b4adfb1d07e2fc004a139 is 175, key is test_row_0/A:col10/1732371942759/Put/seqid=0 2024-11-23T14:25:43,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742001_1177 (size=31055) 2024-11-23T14:25:43,942 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=132, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/A/c2e10ffca30b4adfb1d07e2fc004a139 2024-11-23T14:25:43,955 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/B/e298e2b41c414ca3a59ae1e621b3255e is 50, key is test_row_0/B:col10/1732371942759/Put/seqid=0 2024-11-23T14:25:43,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742002_1178 (size=12101) 2024-11-23T14:25:43,970 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/B/e298e2b41c414ca3a59ae1e621b3255e 2024-11-23T14:25:43,979 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/C/4ff49d8a7f6f4bc98a09c8b88609e86e is 50, key is test_row_0/C:col10/1732371942759/Put/seqid=0 2024-11-23T14:25:43,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742003_1179 (size=12101) 2024-11-23T14:25:43,987 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/C/4ff49d8a7f6f4bc98a09c8b88609e86e 2024-11-23T14:25:44,000 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/A/c2e10ffca30b4adfb1d07e2fc004a139 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/c2e10ffca30b4adfb1d07e2fc004a139 2024-11-23T14:25:44,016 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/c2e10ffca30b4adfb1d07e2fc004a139, entries=150, sequenceid=132, filesize=30.3 K 2024-11-23T14:25:44,020 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/B/e298e2b41c414ca3a59ae1e621b3255e as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/e298e2b41c414ca3a59ae1e621b3255e 2024-11-23T14:25:44,027 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/e298e2b41c414ca3a59ae1e621b3255e, entries=150, sequenceid=132, filesize=11.8 K 2024-11-23T14:25:44,028 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/C/4ff49d8a7f6f4bc98a09c8b88609e86e as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/4ff49d8a7f6f4bc98a09c8b88609e86e 2024-11-23T14:25:44,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:44,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40476 deadline: 1732372004022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:44,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:44,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:44,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:44,036 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/4ff49d8a7f6f4bc98a09c8b88609e86e, entries=150, sequenceid=132, filesize=11.8 K 2024-11-23T14:25:44,037 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for a36a6f46a23b82a88d429516318dcd05 in 135ms, sequenceid=132, compaction requested=true 2024-11-23T14:25:44,037 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a36a6f46a23b82a88d429516318dcd05: 2024-11-23T14:25:44,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a36a6f46a23b82a88d429516318dcd05:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T14:25:44,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:25:44,037 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T14:25:44,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a36a6f46a23b82a88d429516318dcd05:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T14:25:44,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:25:44,037 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T14:25:44,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a36a6f46a23b82a88d429516318dcd05:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T14:25:44,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:25:44,039 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48242 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T14:25:44,039 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141242 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T14:25:44,039 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1541): a36a6f46a23b82a88d429516318dcd05/A is initiating minor compaction (all files) 2024-11-23T14:25:44,039 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1541): a36a6f46a23b82a88d429516318dcd05/B is initiating minor compaction (all files) 2024-11-23T14:25:44,039 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a36a6f46a23b82a88d429516318dcd05/B in TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:44,039 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a36a6f46a23b82a88d429516318dcd05/A in TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:44,039 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/a76a96cbec9b4d838c5f575d3708c9d7, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/dd868ed7718c4340a463cc0c6951ff27, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/8f12c3e5ecbb49d49e15a181521e0a05, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/c2e10ffca30b4adfb1d07e2fc004a139] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp, totalSize=137.9 K 2024-11-23T14:25:44,039 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/e080fdca63e84552ad50f80a6ce3d44a, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/500ec3da77a34ecbbba528b6865f204b, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/35ac91d05fab48729159f47a56d86816, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/e298e2b41c414ca3a59ae1e621b3255e] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp, totalSize=47.1 K 2024-11-23T14:25:44,039 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:44,039 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. files: [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/a76a96cbec9b4d838c5f575d3708c9d7, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/dd868ed7718c4340a463cc0c6951ff27, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/8f12c3e5ecbb49d49e15a181521e0a05, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/c2e10ffca30b4adfb1d07e2fc004a139] 2024-11-23T14:25:44,040 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting e080fdca63e84552ad50f80a6ce3d44a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1732371940530 2024-11-23T14:25:44,040 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting a76a96cbec9b4d838c5f575d3708c9d7, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1732371940530 2024-11-23T14:25:44,040 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 500ec3da77a34ecbbba528b6865f204b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732371941433 2024-11-23T14:25:44,041 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd868ed7718c4340a463cc0c6951ff27, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732371940694 2024-11-23T14:25:44,041 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 35ac91d05fab48729159f47a56d86816, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1732371941585 2024-11-23T14:25:44,041 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8f12c3e5ecbb49d49e15a181521e0a05, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1732371941585 2024-11-23T14:25:44,041 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting e298e2b41c414ca3a59ae1e621b3255e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732371942756 2024-11-23T14:25:44,042 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting c2e10ffca30b4adfb1d07e2fc004a139, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732371942756 2024-11-23T14:25:44,068 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a36a6f46a23b82a88d429516318dcd05#B#compaction#156 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:25:44,068 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/B/f12d54b6303c440da62eeaef5075cd73 is 50, key is test_row_0/B:col10/1732371942759/Put/seqid=0 2024-11-23T14:25:44,073 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a36a6f46a23b82a88d429516318dcd05] 2024-11-23T14:25:44,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742004_1180 (size=12375) 2024-11-23T14:25:44,089 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411239972cddae42b40b8802801999480c97b_a36a6f46a23b82a88d429516318dcd05 store=[table=TestAcidGuarantees family=A region=a36a6f46a23b82a88d429516318dcd05] 2024-11-23T14:25:44,092 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/B/f12d54b6303c440da62eeaef5075cd73 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/f12d54b6303c440da62eeaef5075cd73 2024-11-23T14:25:44,092 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411239972cddae42b40b8802801999480c97b_a36a6f46a23b82a88d429516318dcd05, store=[table=TestAcidGuarantees family=A region=a36a6f46a23b82a88d429516318dcd05] 2024-11-23T14:25:44,092 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411239972cddae42b40b8802801999480c97b_a36a6f46a23b82a88d429516318dcd05 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a36a6f46a23b82a88d429516318dcd05] 2024-11-23T14:25:44,127 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a36a6f46a23b82a88d429516318dcd05/B of a36a6f46a23b82a88d429516318dcd05 into f12d54b6303c440da62eeaef5075cd73(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:25:44,127 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a36a6f46a23b82a88d429516318dcd05: 2024-11-23T14:25:44,127 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., storeName=a36a6f46a23b82a88d429516318dcd05/B, priority=12, startTime=1732371944037; duration=0sec 2024-11-23T14:25:44,127 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:25:44,127 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a36a6f46a23b82a88d429516318dcd05:B 2024-11-23T14:25:44,127 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T14:25:44,129 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48242 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T14:25:44,129 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1541): a36a6f46a23b82a88d429516318dcd05/C is initiating minor compaction (all files) 2024-11-23T14:25:44,129 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a36a6f46a23b82a88d429516318dcd05/C in TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:44,129 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/78154b6552814181a3a53575262732eb, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/f5070d71c7104d53bedd73258d5d022b, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/0f3695881b19489faba6a663bf85188f, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/4ff49d8a7f6f4bc98a09c8b88609e86e] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp, totalSize=47.1 K 2024-11-23T14:25:44,129 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 78154b6552814181a3a53575262732eb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1732371940530 2024-11-23T14:25:44,130 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting f5070d71c7104d53bedd73258d5d022b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732371941433 2024-11-23T14:25:44,130 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f3695881b19489faba6a663bf85188f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1732371941585 2024-11-23T14:25:44,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742005_1181 (size=4469) 2024-11-23T14:25:44,167 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ff49d8a7f6f4bc98a09c8b88609e86e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732371942756 2024-11-23T14:25:44,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on a36a6f46a23b82a88d429516318dcd05 2024-11-23T14:25:44,172 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T14:25:44,172 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a36a6f46a23b82a88d429516318dcd05 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-23T14:25:44,174 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a36a6f46a23b82a88d429516318dcd05#A#compaction#157 average throughput is 0.24 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:25:44,174 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/A/8cd21b4193b243a1b8d54124c9f87f94 is 175, key is test_row_0/A:col10/1732371942759/Put/seqid=0 2024-11-23T14:25:44,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742006_1182 (size=31329) 2024-11-23T14:25:44,207 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/A/8cd21b4193b243a1b8d54124c9f87f94 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/8cd21b4193b243a1b8d54124c9f87f94 2024-11-23T14:25:44,216 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123e328dbd1f0fc475e9f2377e11eb37a59_a36a6f46a23b82a88d429516318dcd05 is 50, key is test_row_0/A:col10/1732371943994/Put/seqid=0 2024-11-23T14:25:44,225 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a36a6f46a23b82a88d429516318dcd05/A of a36a6f46a23b82a88d429516318dcd05 into 8cd21b4193b243a1b8d54124c9f87f94(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:25:44,225 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a36a6f46a23b82a88d429516318dcd05: 2024-11-23T14:25:44,225 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., storeName=a36a6f46a23b82a88d429516318dcd05/A, priority=12, startTime=1732371944037; duration=0sec 2024-11-23T14:25:44,225 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:25:44,225 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a36a6f46a23b82a88d429516318dcd05:A 2024-11-23T14:25:44,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:44,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40476 deadline: 1732372004221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:44,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:44,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:44,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:44,233 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a36a6f46a23b82a88d429516318dcd05#C#compaction#159 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:25:44,234 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/C/0d438be2c0d740f79ac3fe706ccb090e is 50, key is test_row_0/C:col10/1732371942759/Put/seqid=0 2024-11-23T14:25:44,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742007_1183 (size=14794) 2024-11-23T14:25:44,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742008_1184 (size=12375) 2024-11-23T14:25:44,276 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/C/0d438be2c0d740f79ac3fe706ccb090e as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/0d438be2c0d740f79ac3fe706ccb090e 2024-11-23T14:25:44,287 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a36a6f46a23b82a88d429516318dcd05/C of a36a6f46a23b82a88d429516318dcd05 into 0d438be2c0d740f79ac3fe706ccb090e(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:25:44,288 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a36a6f46a23b82a88d429516318dcd05: 2024-11-23T14:25:44,288 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., storeName=a36a6f46a23b82a88d429516318dcd05/C, priority=12, startTime=1732371944037; duration=0sec 2024-11-23T14:25:44,288 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:25:44,288 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a36a6f46a23b82a88d429516318dcd05:C 2024-11-23T14:25:44,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:44,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40476 deadline: 1732372004331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:44,332 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:44,332 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:44,332 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:44,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-23T14:25:44,500 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T14:25:44,501 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-23T14:25:44,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-23T14:25:44,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-23T14:25:44,502 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T14:25:44,503 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T14:25:44,503 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T14:25:44,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:44,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40476 deadline: 1732372004540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:44,542 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:44,542 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:44,542 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:44,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-23T14:25:44,649 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:44,655 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-23T14:25:44,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:44,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. as already flushing 2024-11-23T14:25:44,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:44,656 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:44,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:44,656 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123e328dbd1f0fc475e9f2377e11eb37a59_a36a6f46a23b82a88d429516318dcd05 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123e328dbd1f0fc475e9f2377e11eb37a59_a36a6f46a23b82a88d429516318dcd05 2024-11-23T14:25:44,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:44,657 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/A/e87d76f6e9d74cb2bd43dda13aa2ee23, store: [table=TestAcidGuarantees family=A region=a36a6f46a23b82a88d429516318dcd05] 2024-11-23T14:25:44,658 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/A/e87d76f6e9d74cb2bd43dda13aa2ee23 is 175, key is test_row_0/A:col10/1732371943994/Put/seqid=0 2024-11-23T14:25:44,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742009_1185 (size=39749) 2024-11-23T14:25:44,809 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-23T14:25:44,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:44,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. as already flushing 2024-11-23T14:25:44,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:44,809 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:44,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:44,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:44,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-23T14:25:44,852 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:44,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40476 deadline: 1732372004850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:44,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:44,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:44,853 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:44,961 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-23T14:25:44,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:44,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. as already flushing 2024-11-23T14:25:44,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:44,962 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:44,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:44,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:45,064 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=158, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/A/e87d76f6e9d74cb2bd43dda13aa2ee23 2024-11-23T14:25:45,073 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/B/d49f6b5d2b56432abcfc0f52e42db37a is 50, key is test_row_0/B:col10/1732371943994/Put/seqid=0 2024-11-23T14:25:45,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742010_1186 (size=12151) 2024-11-23T14:25:45,114 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-23T14:25:45,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:45,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. as already flushing 2024-11-23T14:25:45,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:45,115 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:45,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:45,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:45,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-23T14:25:45,267 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-23T14:25:45,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:45,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. as already flushing 2024-11-23T14:25:45,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:45,267 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:45,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:45,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:45,363 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:45,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40476 deadline: 1732372005360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:45,364 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:45,364 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:45,364 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:45,420 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-23T14:25:45,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:45,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. as already flushing 2024-11-23T14:25:45,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:45,420 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:45,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:45,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:45,487 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/B/d49f6b5d2b56432abcfc0f52e42db37a 2024-11-23T14:25:45,498 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/C/0c82664b137145c6b121447f5a621d19 is 50, key is test_row_0/C:col10/1732371943994/Put/seqid=0 2024-11-23T14:25:45,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742011_1187 (size=12151) 2024-11-23T14:25:45,524 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/C/0c82664b137145c6b121447f5a621d19 2024-11-23T14:25:45,532 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/A/e87d76f6e9d74cb2bd43dda13aa2ee23 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/e87d76f6e9d74cb2bd43dda13aa2ee23 2024-11-23T14:25:45,539 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/e87d76f6e9d74cb2bd43dda13aa2ee23, entries=200, sequenceid=158, filesize=38.8 K 2024-11-23T14:25:45,540 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/B/d49f6b5d2b56432abcfc0f52e42db37a as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/d49f6b5d2b56432abcfc0f52e42db37a 2024-11-23T14:25:45,547 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/d49f6b5d2b56432abcfc0f52e42db37a, entries=150, sequenceid=158, filesize=11.9 K 2024-11-23T14:25:45,551 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/C/0c82664b137145c6b121447f5a621d19 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/0c82664b137145c6b121447f5a621d19 2024-11-23T14:25:45,558 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/0c82664b137145c6b121447f5a621d19, entries=150, sequenceid=158, filesize=11.9 K 2024-11-23T14:25:45,560 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for a36a6f46a23b82a88d429516318dcd05 in 1387ms, sequenceid=158, compaction requested=false 2024-11-23T14:25:45,560 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a36a6f46a23b82a88d429516318dcd05: 2024-11-23T14:25:45,573 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-23T14:25:45,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:45,574 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing a36a6f46a23b82a88d429516318dcd05 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-23T14:25:45,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123da88f18ef85444d2a853b6ca319ce5ae_a36a6f46a23b82a88d429516318dcd05 is 50, key is test_row_0/A:col10/1732371944213/Put/seqid=0 2024-11-23T14:25:45,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742012_1188 (size=12304) 2024-11-23T14:25:45,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-23T14:25:46,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:46,010 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123da88f18ef85444d2a853b6ca319ce5ae_a36a6f46a23b82a88d429516318dcd05 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123da88f18ef85444d2a853b6ca319ce5ae_a36a6f46a23b82a88d429516318dcd05 2024-11-23T14:25:46,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/A/f40a29d0dc524157bb22e01bb4739ee6, store: [table=TestAcidGuarantees family=A region=a36a6f46a23b82a88d429516318dcd05] 2024-11-23T14:25:46,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/A/f40a29d0dc524157bb22e01bb4739ee6 is 175, key is test_row_0/A:col10/1732371944213/Put/seqid=0 2024-11-23T14:25:46,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742013_1189 (size=31105) 2024-11-23T14:25:46,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on a36a6f46a23b82a88d429516318dcd05 2024-11-23T14:25:46,382 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. as already flushing 2024-11-23T14:25:46,416 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=171, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/A/f40a29d0dc524157bb22e01bb4739ee6 2024-11-23T14:25:46,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/B/619081c3ebff40e8b5b372900f9db32d is 50, key is test_row_0/B:col10/1732371944213/Put/seqid=0 2024-11-23T14:25:46,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742014_1190 (size=12151) 2024-11-23T14:25:46,442 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/B/619081c3ebff40e8b5b372900f9db32d 2024-11-23T14:25:46,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/C/c0f9708b31ea47eaa535684a6063474e is 50, key is test_row_0/C:col10/1732371944213/Put/seqid=0 2024-11-23T14:25:46,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:46,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40476 deadline: 1732372006466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:46,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:46,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:46,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:46,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742015_1191 (size=12151) 2024-11-23T14:25:46,581 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:46,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40476 deadline: 1732372006580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:46,582 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:46,582 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:46,582 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:46,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-23T14:25:46,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:46,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40476 deadline: 1732372006790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:46,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:46,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:46,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:46,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:46,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40464 deadline: 1732372006800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:46,802 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to c6f7b1897123,35385,1732371902767 for 'test_row_2' in a36a6f46a23b82a88d429516318dcd05 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-23T14:25:46,803 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:46,803 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:46,803 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:46,811 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:46,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40478 deadline: 1732372006810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:46,811 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to c6f7b1897123,35385,1732371902767 for 'test_row_0' in a36a6f46a23b82a88d429516318dcd05 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-23T14:25:46,812 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:46,812 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:46,812 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:46,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:46,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40436 deadline: 1732372006840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:46,841 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to c6f7b1897123,35385,1732371902767 for 'test_row_0' in a36a6f46a23b82a88d429516318dcd05 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8228 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-23T14:25:46,841 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:46,842 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:46,842 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:46,863 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:46,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40438 deadline: 1732372006860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:46,864 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to c6f7b1897123,35385,1732371902767 for 'test_row_0' in a36a6f46a23b82a88d429516318dcd05 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8247 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-23T14:25:46,864 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:46,864 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:46,864 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:46,876 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/C/c0f9708b31ea47eaa535684a6063474e 2024-11-23T14:25:46,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/A/f40a29d0dc524157bb22e01bb4739ee6 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/f40a29d0dc524157bb22e01bb4739ee6 2024-11-23T14:25:46,886 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/f40a29d0dc524157bb22e01bb4739ee6, entries=150, sequenceid=171, filesize=30.4 K 2024-11-23T14:25:46,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/B/619081c3ebff40e8b5b372900f9db32d as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/619081c3ebff40e8b5b372900f9db32d 2024-11-23T14:25:46,893 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/619081c3ebff40e8b5b372900f9db32d, entries=150, sequenceid=171, filesize=11.9 K 2024-11-23T14:25:46,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/C/c0f9708b31ea47eaa535684a6063474e as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/c0f9708b31ea47eaa535684a6063474e 2024-11-23T14:25:46,902 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/c0f9708b31ea47eaa535684a6063474e, entries=150, sequenceid=171, filesize=11.9 K 2024-11-23T14:25:46,903 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for a36a6f46a23b82a88d429516318dcd05 in 1329ms, sequenceid=171, compaction requested=true 2024-11-23T14:25:46,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for a36a6f46a23b82a88d429516318dcd05: 2024-11-23T14:25:46,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:46,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-23T14:25:46,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-23T14:25:46,907 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-23T14:25:46,907 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4020 sec 2024-11-23T14:25:46,909 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 2.4060 sec 2024-11-23T14:25:47,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on a36a6f46a23b82a88d429516318dcd05 2024-11-23T14:25:47,102 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T14:25:47,102 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a36a6f46a23b82a88d429516318dcd05 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-23T14:25:47,114 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112356e51ef1ad11499a9eb686167d22a5c5_a36a6f46a23b82a88d429516318dcd05 is 50, key is test_row_0/A:col10/1732371946452/Put/seqid=0 2024-11-23T14:25:47,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742016_1192 (size=14794) 2024-11-23T14:25:47,129 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:47,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40476 deadline: 1732372007126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:47,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:47,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:47,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:47,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:47,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40476 deadline: 1732372007240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:47,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:47,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:47,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:47,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:47,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40476 deadline: 1732372007450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:47,453 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:47,453 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:47,453 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:47,522 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:47,526 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112356e51ef1ad11499a9eb686167d22a5c5_a36a6f46a23b82a88d429516318dcd05 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112356e51ef1ad11499a9eb686167d22a5c5_a36a6f46a23b82a88d429516318dcd05 2024-11-23T14:25:47,526 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/A/c0a08f3c5ad442eeb911ec04481b9f59, store: [table=TestAcidGuarantees family=A region=a36a6f46a23b82a88d429516318dcd05] 2024-11-23T14:25:47,527 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/A/c0a08f3c5ad442eeb911ec04481b9f59 is 175, key is test_row_0/A:col10/1732371946452/Put/seqid=0 2024-11-23T14:25:47,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742017_1193 (size=39749) 2024-11-23T14:25:47,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:47,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40476 deadline: 1732372007760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:47,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:47,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:47,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:47,931 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=197, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/A/c0a08f3c5ad442eeb911ec04481b9f59 2024-11-23T14:25:47,939 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/B/dd3168c6ce1f42538dd1e952fcbc5ac9 is 50, key is test_row_0/B:col10/1732371946452/Put/seqid=0 2024-11-23T14:25:47,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742018_1194 (size=12151) 2024-11-23T14:25:48,040 INFO [master/c6f7b1897123:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-23T14:25:48,040 INFO [master/c6f7b1897123:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-23T14:25:48,272 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:48,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40476 deadline: 1732372008270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:48,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:48,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:48,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:48,345 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/B/dd3168c6ce1f42538dd1e952fcbc5ac9 2024-11-23T14:25:48,352 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/C/3042e046242f4acc83806cb5c3ee4d9d is 50, key is test_row_0/C:col10/1732371946452/Put/seqid=0 2024-11-23T14:25:48,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742019_1195 (size=12151) 2024-11-23T14:25:48,361 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/C/3042e046242f4acc83806cb5c3ee4d9d 2024-11-23T14:25:48,366 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/A/c0a08f3c5ad442eeb911ec04481b9f59 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/c0a08f3c5ad442eeb911ec04481b9f59 2024-11-23T14:25:48,372 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/c0a08f3c5ad442eeb911ec04481b9f59, entries=200, sequenceid=197, filesize=38.8 K 2024-11-23T14:25:48,374 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/B/dd3168c6ce1f42538dd1e952fcbc5ac9 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/dd3168c6ce1f42538dd1e952fcbc5ac9 2024-11-23T14:25:48,377 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/dd3168c6ce1f42538dd1e952fcbc5ac9, entries=150, sequenceid=197, filesize=11.9 K 2024-11-23T14:25:48,379 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/C/3042e046242f4acc83806cb5c3ee4d9d as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/3042e046242f4acc83806cb5c3ee4d9d 2024-11-23T14:25:48,382 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/3042e046242f4acc83806cb5c3ee4d9d, entries=150, sequenceid=197, filesize=11.9 K 2024-11-23T14:25:48,384 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for a36a6f46a23b82a88d429516318dcd05 in 1281ms, sequenceid=197, compaction requested=true 2024-11-23T14:25:48,384 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a36a6f46a23b82a88d429516318dcd05: 2024-11-23T14:25:48,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a36a6f46a23b82a88d429516318dcd05:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T14:25:48,384 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T14:25:48,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:25:48,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a36a6f46a23b82a88d429516318dcd05:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T14:25:48,384 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T14:25:48,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:25:48,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a36a6f46a23b82a88d429516318dcd05:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T14:25:48,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:25:48,385 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141932 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T14:25:48,385 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48828 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T14:25:48,385 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1541): a36a6f46a23b82a88d429516318dcd05/B is initiating minor compaction (all files) 2024-11-23T14:25:48,385 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1541): a36a6f46a23b82a88d429516318dcd05/A is initiating minor compaction (all files) 2024-11-23T14:25:48,385 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a36a6f46a23b82a88d429516318dcd05/A in TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:48,385 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a36a6f46a23b82a88d429516318dcd05/B in TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:48,385 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/8cd21b4193b243a1b8d54124c9f87f94, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/e87d76f6e9d74cb2bd43dda13aa2ee23, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/f40a29d0dc524157bb22e01bb4739ee6, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/c0a08f3c5ad442eeb911ec04481b9f59] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp, totalSize=138.6 K 2024-11-23T14:25:48,385 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/f12d54b6303c440da62eeaef5075cd73, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/d49f6b5d2b56432abcfc0f52e42db37a, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/619081c3ebff40e8b5b372900f9db32d, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/dd3168c6ce1f42538dd1e952fcbc5ac9] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp, totalSize=47.7 K 2024-11-23T14:25:48,385 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:48,385 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. files: [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/8cd21b4193b243a1b8d54124c9f87f94, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/e87d76f6e9d74cb2bd43dda13aa2ee23, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/f40a29d0dc524157bb22e01bb4739ee6, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/c0a08f3c5ad442eeb911ec04481b9f59] 2024-11-23T14:25:48,386 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting f12d54b6303c440da62eeaef5075cd73, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732371942756 2024-11-23T14:25:48,386 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8cd21b4193b243a1b8d54124c9f87f94, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732371942756 2024-11-23T14:25:48,386 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting d49f6b5d2b56432abcfc0f52e42db37a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1732371943994 2024-11-23T14:25:48,386 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting e87d76f6e9d74cb2bd43dda13aa2ee23, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1732371943994 2024-11-23T14:25:48,386 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 619081c3ebff40e8b5b372900f9db32d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732371944205 2024-11-23T14:25:48,387 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting f40a29d0dc524157bb22e01bb4739ee6, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732371944205 2024-11-23T14:25:48,387 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting dd3168c6ce1f42538dd1e952fcbc5ac9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1732371946452 2024-11-23T14:25:48,387 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting c0a08f3c5ad442eeb911ec04481b9f59, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1732371946452 2024-11-23T14:25:48,397 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a36a6f46a23b82a88d429516318dcd05] 2024-11-23T14:25:48,397 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a36a6f46a23b82a88d429516318dcd05#B#compaction#168 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:25:48,398 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/B/6efefc8cc91e439baccd99ec02219605 is 50, key is test_row_0/B:col10/1732371946452/Put/seqid=0 2024-11-23T14:25:48,399 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112390cf7a1363d1438d89f94c626f083fb5_a36a6f46a23b82a88d429516318dcd05 store=[table=TestAcidGuarantees family=A region=a36a6f46a23b82a88d429516318dcd05] 2024-11-23T14:25:48,401 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112390cf7a1363d1438d89f94c626f083fb5_a36a6f46a23b82a88d429516318dcd05, store=[table=TestAcidGuarantees family=A region=a36a6f46a23b82a88d429516318dcd05] 2024-11-23T14:25:48,401 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112390cf7a1363d1438d89f94c626f083fb5_a36a6f46a23b82a88d429516318dcd05 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a36a6f46a23b82a88d429516318dcd05] 2024-11-23T14:25:48,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742021_1197 (size=4469) 2024-11-23T14:25:48,409 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a36a6f46a23b82a88d429516318dcd05#A#compaction#169 average throughput is 2.04 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:25:48,410 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/A/147ec2f106714d12983bac7983ff0a5f is 175, key is test_row_0/A:col10/1732371946452/Put/seqid=0 2024-11-23T14:25:48,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742020_1196 (size=12561) 2024-11-23T14:25:48,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742022_1198 (size=31515) 2024-11-23T14:25:48,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-23T14:25:48,671 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T14:25:48,671 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-23T14:25:48,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-23T14:25:48,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-23T14:25:48,673 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T14:25:48,674 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T14:25:48,674 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T14:25:48,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-23T14:25:48,826 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-23T14:25:48,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:48,827 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing a36a6f46a23b82a88d429516318dcd05 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-23T14:25:48,831 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/B/6efefc8cc91e439baccd99ec02219605 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/6efefc8cc91e439baccd99ec02219605 2024-11-23T14:25:48,837 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/A/147ec2f106714d12983bac7983ff0a5f as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/147ec2f106714d12983bac7983ff0a5f 2024-11-23T14:25:48,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112320358cbdf6eb450cb14cef83b34cc8aa_a36a6f46a23b82a88d429516318dcd05 is 50, key is test_row_0/A:col10/1732371947124/Put/seqid=0 2024-11-23T14:25:48,842 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a36a6f46a23b82a88d429516318dcd05/B of a36a6f46a23b82a88d429516318dcd05 into 6efefc8cc91e439baccd99ec02219605(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:25:48,843 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a36a6f46a23b82a88d429516318dcd05: 2024-11-23T14:25:48,843 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., storeName=a36a6f46a23b82a88d429516318dcd05/B, priority=12, startTime=1732371948384; duration=0sec 2024-11-23T14:25:48,843 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:25:48,843 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a36a6f46a23b82a88d429516318dcd05:B 2024-11-23T14:25:48,843 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T14:25:48,845 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a36a6f46a23b82a88d429516318dcd05/A of a36a6f46a23b82a88d429516318dcd05 into 147ec2f106714d12983bac7983ff0a5f(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:25:48,845 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a36a6f46a23b82a88d429516318dcd05: 2024-11-23T14:25:48,845 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48828 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T14:25:48,845 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., storeName=a36a6f46a23b82a88d429516318dcd05/A, priority=12, startTime=1732371948384; duration=0sec 2024-11-23T14:25:48,845 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1541): a36a6f46a23b82a88d429516318dcd05/C is initiating minor compaction (all files) 2024-11-23T14:25:48,845 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a36a6f46a23b82a88d429516318dcd05/C in TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:48,845 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:25:48,845 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a36a6f46a23b82a88d429516318dcd05:A 2024-11-23T14:25:48,845 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/0d438be2c0d740f79ac3fe706ccb090e, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/0c82664b137145c6b121447f5a621d19, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/c0f9708b31ea47eaa535684a6063474e, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/3042e046242f4acc83806cb5c3ee4d9d] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp, totalSize=47.7 K 2024-11-23T14:25:48,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742023_1199 (size=12304) 2024-11-23T14:25:48,848 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d438be2c0d740f79ac3fe706ccb090e, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732371942756 2024-11-23T14:25:48,849 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c82664b137145c6b121447f5a621d19, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1732371943994 2024-11-23T14:25:48,849 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting c0f9708b31ea47eaa535684a6063474e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732371944205 2024-11-23T14:25:48,849 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 3042e046242f4acc83806cb5c3ee4d9d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1732371946452 2024-11-23T14:25:48,859 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a36a6f46a23b82a88d429516318dcd05#C#compaction#171 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:25:48,859 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/C/8eb57403117649b8aabb9630efd6fc49 is 50, key is test_row_0/C:col10/1732371946452/Put/seqid=0 2024-11-23T14:25:48,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742024_1200 (size=12561) 2024-11-23T14:25:48,870 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/C/8eb57403117649b8aabb9630efd6fc49 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/8eb57403117649b8aabb9630efd6fc49 2024-11-23T14:25:48,878 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a36a6f46a23b82a88d429516318dcd05/C of a36a6f46a23b82a88d429516318dcd05 into 8eb57403117649b8aabb9630efd6fc49(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:25:48,878 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a36a6f46a23b82a88d429516318dcd05: 2024-11-23T14:25:48,878 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., storeName=a36a6f46a23b82a88d429516318dcd05/C, priority=12, startTime=1732371948384; duration=0sec 2024-11-23T14:25:48,879 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:25:48,879 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a36a6f46a23b82a88d429516318dcd05:C 2024-11-23T14:25:48,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-23T14:25:49,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:49,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on a36a6f46a23b82a88d429516318dcd05 2024-11-23T14:25:49,283 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. as already flushing 2024-11-23T14:25:49,285 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112320358cbdf6eb450cb14cef83b34cc8aa_a36a6f46a23b82a88d429516318dcd05 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112320358cbdf6eb450cb14cef83b34cc8aa_a36a6f46a23b82a88d429516318dcd05 2024-11-23T14:25:49,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/A/fb5bc6e9d42e4b44b5ccfa22af70e77e, store: [table=TestAcidGuarantees family=A region=a36a6f46a23b82a88d429516318dcd05] 2024-11-23T14:25:49,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/A/fb5bc6e9d42e4b44b5ccfa22af70e77e is 175, key is test_row_0/A:col10/1732371947124/Put/seqid=0 2024-11-23T14:25:49,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742025_1201 (size=31105) 2024-11-23T14:25:49,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-23T14:25:49,361 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:49,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40476 deadline: 1732372009357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:49,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:49,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:49,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:49,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:49,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40476 deadline: 1732372009469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:49,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:49,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:49,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:49,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:49,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40476 deadline: 1732372009680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:49,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:49,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:49,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:49,694 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=207, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/A/fb5bc6e9d42e4b44b5ccfa22af70e77e 2024-11-23T14:25:49,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/B/63c3091562c64c44a32fca07a902fb5c is 50, key is test_row_0/B:col10/1732371947124/Put/seqid=0 2024-11-23T14:25:49,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742026_1202 (size=12151) 2024-11-23T14:25:49,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-23T14:25:49,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:49,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40476 deadline: 1732372009990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:49,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:49,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:49,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:50,106 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/B/63c3091562c64c44a32fca07a902fb5c 2024-11-23T14:25:50,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/C/80ecf0bd979847e3944daf3b58a4bcb5 is 50, key is test_row_0/C:col10/1732371947124/Put/seqid=0 2024-11-23T14:25:50,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742027_1203 (size=12151) 2024-11-23T14:25:50,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:50,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40476 deadline: 1732372010500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:50,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:50,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:50,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:50,520 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/C/80ecf0bd979847e3944daf3b58a4bcb5 2024-11-23T14:25:50,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/A/fb5bc6e9d42e4b44b5ccfa22af70e77e as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/fb5bc6e9d42e4b44b5ccfa22af70e77e 2024-11-23T14:25:50,529 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/fb5bc6e9d42e4b44b5ccfa22af70e77e, entries=150, sequenceid=207, filesize=30.4 K 2024-11-23T14:25:50,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/B/63c3091562c64c44a32fca07a902fb5c as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/63c3091562c64c44a32fca07a902fb5c 2024-11-23T14:25:50,533 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/63c3091562c64c44a32fca07a902fb5c, entries=150, sequenceid=207, filesize=11.9 K 2024-11-23T14:25:50,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/C/80ecf0bd979847e3944daf3b58a4bcb5 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/80ecf0bd979847e3944daf3b58a4bcb5 2024-11-23T14:25:50,541 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/80ecf0bd979847e3944daf3b58a4bcb5, entries=150, sequenceid=207, filesize=11.9 K 2024-11-23T14:25:50,542 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for a36a6f46a23b82a88d429516318dcd05 in 1715ms, sequenceid=207, compaction requested=false 2024-11-23T14:25:50,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for a36a6f46a23b82a88d429516318dcd05: 2024-11-23T14:25:50,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:50,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-23T14:25:50,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-23T14:25:50,545 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-23T14:25:50,545 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8690 sec 2024-11-23T14:25:50,546 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 1.8750 sec 2024-11-23T14:25:50,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-23T14:25:50,821 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T14:25:50,821 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-23T14:25:50,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-23T14:25:50,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-23T14:25:50,822 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T14:25:50,823 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T14:25:50,823 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T14:25:50,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-23T14:25:50,975 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-23T14:25:50,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:50,975 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing a36a6f46a23b82a88d429516318dcd05 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-23T14:25:50,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123218f4a0ab1ee4164a08f20da7ed863b1_a36a6f46a23b82a88d429516318dcd05 is 50, key is test_row_0/A:col10/1732371949327/Put/seqid=0 2024-11-23T14:25:50,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742028_1204 (size=12304) 2024-11-23T14:25:51,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-23T14:25:51,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:51,394 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123218f4a0ab1ee4164a08f20da7ed863b1_a36a6f46a23b82a88d429516318dcd05 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123218f4a0ab1ee4164a08f20da7ed863b1_a36a6f46a23b82a88d429516318dcd05 2024-11-23T14:25:51,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/A/fa4aa82e41ad4687a3253e110be54e31, store: [table=TestAcidGuarantees family=A region=a36a6f46a23b82a88d429516318dcd05] 2024-11-23T14:25:51,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/A/fa4aa82e41ad4687a3253e110be54e31 is 175, key is test_row_0/A:col10/1732371949327/Put/seqid=0 2024-11-23T14:25:51,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742029_1205 (size=31105) 2024-11-23T14:25:51,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-23T14:25:51,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on a36a6f46a23b82a88d429516318dcd05 2024-11-23T14:25:51,521 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. as already flushing 2024-11-23T14:25:51,539 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:51,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40476 deadline: 1732372011536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:51,539 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:51,539 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:51,539 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:51,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:51,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40476 deadline: 1732372011650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:51,651 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:51,651 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:51,651 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:51,802 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=236, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/A/fa4aa82e41ad4687a3253e110be54e31 2024-11-23T14:25:51,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/B/a316e62662304bba97282115d3bd79d0 is 50, key is test_row_0/B:col10/1732371949327/Put/seqid=0 2024-11-23T14:25:51,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742030_1206 (size=12151) 2024-11-23T14:25:51,819 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/B/a316e62662304bba97282115d3bd79d0 2024-11-23T14:25:51,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/C/e39c0dc2bf67429181e93efad3f33535 is 50, key is test_row_0/C:col10/1732371949327/Put/seqid=0 2024-11-23T14:25:51,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742031_1207 (size=12151) 2024-11-23T14:25:51,860 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:51,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40476 deadline: 1732372011860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:51,861 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:51,861 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:51,861 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:51,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-23T14:25:52,173 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:52,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40476 deadline: 1732372012170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:52,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:52,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:52,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:52,240 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/C/e39c0dc2bf67429181e93efad3f33535 2024-11-23T14:25:52,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/A/fa4aa82e41ad4687a3253e110be54e31 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/fa4aa82e41ad4687a3253e110be54e31 2024-11-23T14:25:52,262 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/fa4aa82e41ad4687a3253e110be54e31, entries=150, sequenceid=236, filesize=30.4 K 2024-11-23T14:25:52,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/B/a316e62662304bba97282115d3bd79d0 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/a316e62662304bba97282115d3bd79d0 2024-11-23T14:25:52,270 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/a316e62662304bba97282115d3bd79d0, entries=150, sequenceid=236, filesize=11.9 K 2024-11-23T14:25:52,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/C/e39c0dc2bf67429181e93efad3f33535 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/e39c0dc2bf67429181e93efad3f33535 2024-11-23T14:25:52,279 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/e39c0dc2bf67429181e93efad3f33535, entries=150, sequenceid=236, filesize=11.9 K 2024-11-23T14:25:52,281 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for a36a6f46a23b82a88d429516318dcd05 in 1306ms, sequenceid=236, compaction requested=true 2024-11-23T14:25:52,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for a36a6f46a23b82a88d429516318dcd05: 2024-11-23T14:25:52,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:52,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-23T14:25:52,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-23T14:25:52,295 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-23T14:25:52,295 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4590 sec 2024-11-23T14:25:52,297 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 1.4750 sec 2024-11-23T14:25:52,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on a36a6f46a23b82a88d429516318dcd05 2024-11-23T14:25:52,683 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T14:25:52,683 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a36a6f46a23b82a88d429516318dcd05 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-23T14:25:52,690 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123817b998ee2524b3e88572cbcf4488b45_a36a6f46a23b82a88d429516318dcd05 is 50, key is test_row_0/A:col10/1732371951520/Put/seqid=0 2024-11-23T14:25:52,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742032_1208 (size=12304) 2024-11-23T14:25:52,768 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:52,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 225 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40476 deadline: 1732372012765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:52,768 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:52,768 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:52,768 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:52,884 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:52,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 226 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40476 deadline: 1732372012880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:52,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:52,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:52,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:52,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-23T14:25:52,970 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T14:25:52,970 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-23T14:25:52,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-11-23T14:25:52,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-23T14:25:52,973 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T14:25:52,974 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T14:25:52,974 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T14:25:53,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-23T14:25:53,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:53,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40476 deadline: 1732372013090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:53,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:53,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:53,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:53,095 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:53,105 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123817b998ee2524b3e88572cbcf4488b45_a36a6f46a23b82a88d429516318dcd05 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123817b998ee2524b3e88572cbcf4488b45_a36a6f46a23b82a88d429516318dcd05 2024-11-23T14:25:53,106 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/A/08fc454bb6184943864685a2a45b064b, store: [table=TestAcidGuarantees family=A region=a36a6f46a23b82a88d429516318dcd05] 2024-11-23T14:25:53,107 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/A/08fc454bb6184943864685a2a45b064b is 175, key is test_row_0/A:col10/1732371951520/Put/seqid=0 2024-11-23T14:25:53,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742033_1209 (size=31105) 2024-11-23T14:25:53,113 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=247, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/A/08fc454bb6184943864685a2a45b064b 2024-11-23T14:25:53,126 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/B/746fd8d4727c4bfbb86c15064ee95055 is 50, key is test_row_0/B:col10/1732371951520/Put/seqid=0 2024-11-23T14:25:53,126 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-23T14:25:53,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:53,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. as already flushing 2024-11-23T14:25:53,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:53,127 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:53,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:53,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:53,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742034_1210 (size=12151) 2024-11-23T14:25:53,143 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/B/746fd8d4727c4bfbb86c15064ee95055 2024-11-23T14:25:53,149 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/C/18037fa41cb24b1686c7de5bef23c550 is 50, key is test_row_0/C:col10/1732371951520/Put/seqid=0 2024-11-23T14:25:53,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742035_1211 (size=12151) 2024-11-23T14:25:53,281 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-23T14:25:53,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:53,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. as already flushing 2024-11-23T14:25:53,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:53,281 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:53,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:53,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:53,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-23T14:25:53,401 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:53,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 228 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40476 deadline: 1732372013400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:53,401 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:53,401 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:53,401 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:53,434 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-23T14:25:53,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:53,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. as already flushing 2024-11-23T14:25:53,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:53,435 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:53,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:53,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:53,562 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/C/18037fa41cb24b1686c7de5bef23c550 2024-11-23T14:25:53,568 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/A/08fc454bb6184943864685a2a45b064b as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/08fc454bb6184943864685a2a45b064b 2024-11-23T14:25:53,572 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/08fc454bb6184943864685a2a45b064b, entries=150, sequenceid=247, filesize=30.4 K 2024-11-23T14:25:53,573 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/B/746fd8d4727c4bfbb86c15064ee95055 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/746fd8d4727c4bfbb86c15064ee95055 2024-11-23T14:25:53,577 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/746fd8d4727c4bfbb86c15064ee95055, entries=150, sequenceid=247, filesize=11.9 K 2024-11-23T14:25:53,578 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/C/18037fa41cb24b1686c7de5bef23c550 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/18037fa41cb24b1686c7de5bef23c550 2024-11-23T14:25:53,582 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/18037fa41cb24b1686c7de5bef23c550, entries=150, sequenceid=247, filesize=11.9 K 2024-11-23T14:25:53,583 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for a36a6f46a23b82a88d429516318dcd05 in 900ms, sequenceid=247, compaction requested=true 2024-11-23T14:25:53,583 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a36a6f46a23b82a88d429516318dcd05: 2024-11-23T14:25:53,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a36a6f46a23b82a88d429516318dcd05:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T14:25:53,583 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T14:25:53,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:25:53,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a36a6f46a23b82a88d429516318dcd05:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T14:25:53,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:25:53,583 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T14:25:53,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a36a6f46a23b82a88d429516318dcd05:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T14:25:53,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:25:53,585 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 124830 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T14:25:53,585 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T14:25:53,585 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1541): a36a6f46a23b82a88d429516318dcd05/B is initiating minor compaction (all files) 2024-11-23T14:25:53,585 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1541): a36a6f46a23b82a88d429516318dcd05/A is initiating minor compaction (all files) 2024-11-23T14:25:53,585 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a36a6f46a23b82a88d429516318dcd05/B in TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:53,585 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a36a6f46a23b82a88d429516318dcd05/A in TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:53,585 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/147ec2f106714d12983bac7983ff0a5f, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/fb5bc6e9d42e4b44b5ccfa22af70e77e, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/fa4aa82e41ad4687a3253e110be54e31, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/08fc454bb6184943864685a2a45b064b] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp, totalSize=121.9 K 2024-11-23T14:25:53,585 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/6efefc8cc91e439baccd99ec02219605, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/63c3091562c64c44a32fca07a902fb5c, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/a316e62662304bba97282115d3bd79d0, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/746fd8d4727c4bfbb86c15064ee95055] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp, totalSize=47.9 K 2024-11-23T14:25:53,585 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:53,585 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. files: [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/147ec2f106714d12983bac7983ff0a5f, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/fb5bc6e9d42e4b44b5ccfa22af70e77e, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/fa4aa82e41ad4687a3253e110be54e31, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/08fc454bb6184943864685a2a45b064b] 2024-11-23T14:25:53,586 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 6efefc8cc91e439baccd99ec02219605, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1732371946452 2024-11-23T14:25:53,586 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 147ec2f106714d12983bac7983ff0a5f, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1732371946452 2024-11-23T14:25:53,586 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb5bc6e9d42e4b44b5ccfa22af70e77e, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1732371947115 2024-11-23T14:25:53,586 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 63c3091562c64c44a32fca07a902fb5c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1732371947115 2024-11-23T14:25:53,586 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting a316e62662304bba97282115d3bd79d0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1732371949327 2024-11-23T14:25:53,587 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa4aa82e41ad4687a3253e110be54e31, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1732371949327 2024-11-23T14:25:53,587 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 746fd8d4727c4bfbb86c15064ee95055, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1732371951520 2024-11-23T14:25:53,587 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 08fc454bb6184943864685a2a45b064b, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1732371951520 2024-11-23T14:25:53,588 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-23T14:25:53,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:53,588 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing a36a6f46a23b82a88d429516318dcd05 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-23T14:25:53,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-23T14:25:53,604 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a36a6f46a23b82a88d429516318dcd05#B#compaction#180 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:25:53,605 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/B/e06d09b41bc64408bdb65b3810afe163 is 50, key is test_row_0/B:col10/1732371951520/Put/seqid=0 2024-11-23T14:25:53,607 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a36a6f46a23b82a88d429516318dcd05] 2024-11-23T14:25:53,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411237b183e6e7b994e2ab82046363b20c54b_a36a6f46a23b82a88d429516318dcd05 is 50, key is test_row_0/A:col10/1732371952719/Put/seqid=0 2024-11-23T14:25:53,629 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411234ac866523624498a9318b3dff60a6ee0_a36a6f46a23b82a88d429516318dcd05 store=[table=TestAcidGuarantees family=A region=a36a6f46a23b82a88d429516318dcd05] 2024-11-23T14:25:53,631 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411234ac866523624498a9318b3dff60a6ee0_a36a6f46a23b82a88d429516318dcd05, store=[table=TestAcidGuarantees family=A region=a36a6f46a23b82a88d429516318dcd05] 2024-11-23T14:25:53,631 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411234ac866523624498a9318b3dff60a6ee0_a36a6f46a23b82a88d429516318dcd05 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a36a6f46a23b82a88d429516318dcd05] 2024-11-23T14:25:53,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742038_1214 (size=4469) 2024-11-23T14:25:53,637 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a36a6f46a23b82a88d429516318dcd05#A#compaction#181 average throughput is 0.81 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:25:53,638 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/A/7a53ec4448be4fb787e2e16614c1631b is 175, key is test_row_0/A:col10/1732371951520/Put/seqid=0 2024-11-23T14:25:53,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742037_1213 (size=12454) 2024-11-23T14:25:53,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:53,647 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411237b183e6e7b994e2ab82046363b20c54b_a36a6f46a23b82a88d429516318dcd05 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411237b183e6e7b994e2ab82046363b20c54b_a36a6f46a23b82a88d429516318dcd05 2024-11-23T14:25:53,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742036_1212 (size=12697) 2024-11-23T14:25:53,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/A/3b705a2fca904b579dc7b56873bc8a3a, store: [table=TestAcidGuarantees family=A region=a36a6f46a23b82a88d429516318dcd05] 2024-11-23T14:25:53,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/A/3b705a2fca904b579dc7b56873bc8a3a is 175, key is test_row_0/A:col10/1732371952719/Put/seqid=0 2024-11-23T14:25:53,660 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/B/e06d09b41bc64408bdb65b3810afe163 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/e06d09b41bc64408bdb65b3810afe163 2024-11-23T14:25:53,666 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a36a6f46a23b82a88d429516318dcd05/B of a36a6f46a23b82a88d429516318dcd05 into e06d09b41bc64408bdb65b3810afe163(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:25:53,666 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a36a6f46a23b82a88d429516318dcd05: 2024-11-23T14:25:53,666 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., storeName=a36a6f46a23b82a88d429516318dcd05/B, priority=12, startTime=1732371953583; duration=0sec 2024-11-23T14:25:53,666 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:25:53,666 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a36a6f46a23b82a88d429516318dcd05:B 2024-11-23T14:25:53,666 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T14:25:53,667 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T14:25:53,668 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1541): a36a6f46a23b82a88d429516318dcd05/C is initiating minor compaction (all files) 2024-11-23T14:25:53,668 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a36a6f46a23b82a88d429516318dcd05/C in TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:53,668 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/8eb57403117649b8aabb9630efd6fc49, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/80ecf0bd979847e3944daf3b58a4bcb5, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/e39c0dc2bf67429181e93efad3f33535, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/18037fa41cb24b1686c7de5bef23c550] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp, totalSize=47.9 K 2024-11-23T14:25:53,668 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 8eb57403117649b8aabb9630efd6fc49, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1732371946452 2024-11-23T14:25:53,669 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 80ecf0bd979847e3944daf3b58a4bcb5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1732371947115 2024-11-23T14:25:53,670 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting e39c0dc2bf67429181e93efad3f33535, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1732371949327 2024-11-23T14:25:53,670 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 18037fa41cb24b1686c7de5bef23c550, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1732371951520 2024-11-23T14:25:53,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742039_1215 (size=31651) 2024-11-23T14:25:53,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742040_1216 (size=31255) 2024-11-23T14:25:53,679 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=272, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/A/3b705a2fca904b579dc7b56873bc8a3a 2024-11-23T14:25:53,684 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a36a6f46a23b82a88d429516318dcd05#C#compaction#183 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:25:53,685 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/C/2cf16f5675ea441286131274963dbf8e is 50, key is test_row_0/C:col10/1732371951520/Put/seqid=0 2024-11-23T14:25:53,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/B/c091790ccaca458aa48931d93c5161d4 is 50, key is test_row_0/B:col10/1732371952719/Put/seqid=0 2024-11-23T14:25:53,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742041_1217 (size=12697) 2024-11-23T14:25:53,704 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/C/2cf16f5675ea441286131274963dbf8e as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/2cf16f5675ea441286131274963dbf8e 2024-11-23T14:25:53,713 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a36a6f46a23b82a88d429516318dcd05/C of a36a6f46a23b82a88d429516318dcd05 into 2cf16f5675ea441286131274963dbf8e(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:25:53,713 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a36a6f46a23b82a88d429516318dcd05: 2024-11-23T14:25:53,713 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., storeName=a36a6f46a23b82a88d429516318dcd05/C, priority=12, startTime=1732371953583; duration=0sec 2024-11-23T14:25:53,714 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:25:53,714 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a36a6f46a23b82a88d429516318dcd05:C 2024-11-23T14:25:53,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742042_1218 (size=12301) 2024-11-23T14:25:53,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on a36a6f46a23b82a88d429516318dcd05 2024-11-23T14:25:53,913 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. as already flushing 2024-11-23T14:25:53,942 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:53,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 237 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40476 deadline: 1732372013940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:53,942 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:53,942 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:53,942 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:54,051 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:54,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 238 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40476 deadline: 1732372014049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:54,052 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:54,052 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:54,052 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:54,079 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/A/7a53ec4448be4fb787e2e16614c1631b as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/7a53ec4448be4fb787e2e16614c1631b 2024-11-23T14:25:54,084 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a36a6f46a23b82a88d429516318dcd05/A of a36a6f46a23b82a88d429516318dcd05 into 7a53ec4448be4fb787e2e16614c1631b(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:25:54,084 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a36a6f46a23b82a88d429516318dcd05: 2024-11-23T14:25:54,084 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., storeName=a36a6f46a23b82a88d429516318dcd05/A, priority=12, startTime=1732371953583; duration=0sec 2024-11-23T14:25:54,085 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:25:54,085 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a36a6f46a23b82a88d429516318dcd05:A 2024-11-23T14:25:54,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-23T14:25:54,119 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/B/c091790ccaca458aa48931d93c5161d4 2024-11-23T14:25:54,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/C/aa54469a03084c8da913c2a8eadbf2aa is 50, key is test_row_0/C:col10/1732371952719/Put/seqid=0 2024-11-23T14:25:54,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742043_1219 (size=12301) 2024-11-23T14:25:54,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:54,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 239 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40476 deadline: 1732372014260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:54,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:54,263 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:54,263 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:54,533 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/C/aa54469a03084c8da913c2a8eadbf2aa 2024-11-23T14:25:54,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/A/3b705a2fca904b579dc7b56873bc8a3a as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/3b705a2fca904b579dc7b56873bc8a3a 2024-11-23T14:25:54,557 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/3b705a2fca904b579dc7b56873bc8a3a, entries=150, sequenceid=272, filesize=30.5 K 2024-11-23T14:25:54,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/B/c091790ccaca458aa48931d93c5161d4 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/c091790ccaca458aa48931d93c5161d4 2024-11-23T14:25:54,566 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/c091790ccaca458aa48931d93c5161d4, entries=150, sequenceid=272, filesize=12.0 K 2024-11-23T14:25:54,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/C/aa54469a03084c8da913c2a8eadbf2aa as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/aa54469a03084c8da913c2a8eadbf2aa 2024-11-23T14:25:54,570 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:54,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 240 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40476 deadline: 1732372014570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:54,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:54,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:54,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:54,572 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/aa54469a03084c8da913c2a8eadbf2aa, entries=150, sequenceid=272, filesize=12.0 K 2024-11-23T14:25:54,573 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for a36a6f46a23b82a88d429516318dcd05 in 985ms, sequenceid=272, compaction requested=false 2024-11-23T14:25:54,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for a36a6f46a23b82a88d429516318dcd05: 2024-11-23T14:25:54,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:54,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-23T14:25:54,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-23T14:25:54,576 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-23T14:25:54,576 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6010 sec 2024-11-23T14:25:54,578 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 1.6060 sec 2024-11-23T14:25:55,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on a36a6f46a23b82a88d429516318dcd05 2024-11-23T14:25:55,082 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T14:25:55,082 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a36a6f46a23b82a88d429516318dcd05 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-23T14:25:55,098 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123e38066d409ab4b0ead37b6f521960db5_a36a6f46a23b82a88d429516318dcd05 is 50, key is test_row_0/A:col10/1732371953923/Put/seqid=0 2024-11-23T14:25:55,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742044_1220 (size=14994) 2024-11-23T14:25:55,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-23T14:25:55,131 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T14:25:55,132 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-23T14:25:55,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees 2024-11-23T14:25:55,136 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T14:25:55,137 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T14:25:55,137 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T14:25:55,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-23T14:25:55,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:55,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 263 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40476 deadline: 1732372015178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:55,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:55,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:55,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:55,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-23T14:25:55,291 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:55,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 264 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40476 deadline: 1732372015290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:55,291 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-23T14:25:55,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:55,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:55,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. as already flushing 2024-11-23T14:25:55,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:55,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:55,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:55,292 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:55,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:55,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:55,445 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-23T14:25:55,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:55,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. as already flushing 2024-11-23T14:25:55,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:55,445 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:55,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:55,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:55,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-23T14:25:55,504 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:55,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:55,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 265 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40476 deadline: 1732372015500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:55,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:55,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:55,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:55,514 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123e38066d409ab4b0ead37b6f521960db5_a36a6f46a23b82a88d429516318dcd05 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123e38066d409ab4b0ead37b6f521960db5_a36a6f46a23b82a88d429516318dcd05 2024-11-23T14:25:55,516 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/A/23e8c6d7e8df48188ca9e7d68665ad05, store: [table=TestAcidGuarantees family=A region=a36a6f46a23b82a88d429516318dcd05] 2024-11-23T14:25:55,516 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/A/23e8c6d7e8df48188ca9e7d68665ad05 is 175, key is test_row_0/A:col10/1732371953923/Put/seqid=0 2024-11-23T14:25:55,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742045_1221 (size=39949) 2024-11-23T14:25:55,521 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=287, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/A/23e8c6d7e8df48188ca9e7d68665ad05 2024-11-23T14:25:55,545 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/B/8056a3cffba1405ab1590e1fb330afb0 is 50, key is test_row_0/B:col10/1732371953923/Put/seqid=0 2024-11-23T14:25:55,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742046_1222 (size=12301) 2024-11-23T14:25:55,598 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-23T14:25:55,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:55,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. as already flushing 2024-11-23T14:25:55,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:55,598 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:55,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:55,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:55,751 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-23T14:25:55,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:55,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. as already flushing 2024-11-23T14:25:55,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:55,751 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:55,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:55,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:55,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-23T14:25:55,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:55,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 266 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40476 deadline: 1732372015810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:55,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:55,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:55,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:55,904 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-23T14:25:55,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:55,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. as already flushing 2024-11-23T14:25:55,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:55,904 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:55,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:55,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:55,969 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/B/8056a3cffba1405ab1590e1fb330afb0 2024-11-23T14:25:55,982 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/C/39508c3260854100a9713cdefcbbd5b9 is 50, key is test_row_0/C:col10/1732371953923/Put/seqid=0 2024-11-23T14:25:56,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742047_1223 (size=12301) 2024-11-23T14:25:56,057 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-23T14:25:56,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:56,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. as already flushing 2024-11-23T14:25:56,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:56,057 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:56,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:56,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:56,210 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-23T14:25:56,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:56,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. as already flushing 2024-11-23T14:25:56,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:56,210 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:56,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:56,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:56,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-23T14:25:56,325 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:56,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 267 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40476 deadline: 1732372016320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:56,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:56,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:56,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:56,363 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-23T14:25:56,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:56,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. as already flushing 2024-11-23T14:25:56,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:56,364 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:56,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:56,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:25:56,413 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/C/39508c3260854100a9713cdefcbbd5b9 2024-11-23T14:25:56,419 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/A/23e8c6d7e8df48188ca9e7d68665ad05 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/23e8c6d7e8df48188ca9e7d68665ad05 2024-11-23T14:25:56,424 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/23e8c6d7e8df48188ca9e7d68665ad05, entries=200, sequenceid=287, filesize=39.0 K 2024-11-23T14:25:56,425 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/B/8056a3cffba1405ab1590e1fb330afb0 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/8056a3cffba1405ab1590e1fb330afb0 2024-11-23T14:25:56,431 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/8056a3cffba1405ab1590e1fb330afb0, entries=150, sequenceid=287, filesize=12.0 K 2024-11-23T14:25:56,432 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/C/39508c3260854100a9713cdefcbbd5b9 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/39508c3260854100a9713cdefcbbd5b9 2024-11-23T14:25:56,437 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/39508c3260854100a9713cdefcbbd5b9, entries=150, sequenceid=287, filesize=12.0 K 2024-11-23T14:25:56,438 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for a36a6f46a23b82a88d429516318dcd05 in 1356ms, sequenceid=287, compaction requested=true 2024-11-23T14:25:56,438 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a36a6f46a23b82a88d429516318dcd05: 2024-11-23T14:25:56,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a36a6f46a23b82a88d429516318dcd05:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T14:25:56,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:25:56,439 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:25:56,439 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:25:56,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a36a6f46a23b82a88d429516318dcd05:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T14:25:56,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:25:56,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a36a6f46a23b82a88d429516318dcd05:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T14:25:56,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:25:56,440 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102855 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:25:56,440 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:25:56,440 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1541): a36a6f46a23b82a88d429516318dcd05/B is initiating minor compaction (all files) 2024-11-23T14:25:56,440 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1541): a36a6f46a23b82a88d429516318dcd05/A is initiating minor compaction (all files) 2024-11-23T14:25:56,440 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a36a6f46a23b82a88d429516318dcd05/B in TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:56,441 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a36a6f46a23b82a88d429516318dcd05/A in TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:56,441 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/7a53ec4448be4fb787e2e16614c1631b, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/3b705a2fca904b579dc7b56873bc8a3a, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/23e8c6d7e8df48188ca9e7d68665ad05] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp, totalSize=100.4 K 2024-11-23T14:25:56,441 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/e06d09b41bc64408bdb65b3810afe163, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/c091790ccaca458aa48931d93c5161d4, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/8056a3cffba1405ab1590e1fb330afb0] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp, totalSize=36.4 K 2024-11-23T14:25:56,441 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:56,441 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. files: [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/7a53ec4448be4fb787e2e16614c1631b, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/3b705a2fca904b579dc7b56873bc8a3a, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/23e8c6d7e8df48188ca9e7d68665ad05] 2024-11-23T14:25:56,441 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting e06d09b41bc64408bdb65b3810afe163, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1732371951520 2024-11-23T14:25:56,441 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a53ec4448be4fb787e2e16614c1631b, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1732371951520 2024-11-23T14:25:56,442 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b705a2fca904b579dc7b56873bc8a3a, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1732371952719 2024-11-23T14:25:56,442 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting c091790ccaca458aa48931d93c5161d4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1732371952719 2024-11-23T14:25:56,442 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 23e8c6d7e8df48188ca9e7d68665ad05, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732371953923 2024-11-23T14:25:56,442 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 8056a3cffba1405ab1590e1fb330afb0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732371953923 2024-11-23T14:25:56,462 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a36a6f46a23b82a88d429516318dcd05#B#compaction#189 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:25:56,463 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/B/febef176354b465fb8683f9a261d65fe is 50, key is test_row_0/B:col10/1732371953923/Put/seqid=0 2024-11-23T14:25:56,473 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a36a6f46a23b82a88d429516318dcd05] 2024-11-23T14:25:56,496 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411237025c36380974247b876f78cc0c9a2ab_a36a6f46a23b82a88d429516318dcd05 store=[table=TestAcidGuarantees family=A region=a36a6f46a23b82a88d429516318dcd05] 2024-11-23T14:25:56,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742048_1224 (size=12949) 2024-11-23T14:25:56,498 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411237025c36380974247b876f78cc0c9a2ab_a36a6f46a23b82a88d429516318dcd05, store=[table=TestAcidGuarantees family=A region=a36a6f46a23b82a88d429516318dcd05] 2024-11-23T14:25:56,498 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411237025c36380974247b876f78cc0c9a2ab_a36a6f46a23b82a88d429516318dcd05 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a36a6f46a23b82a88d429516318dcd05] 2024-11-23T14:25:56,508 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/B/febef176354b465fb8683f9a261d65fe as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/febef176354b465fb8683f9a261d65fe 2024-11-23T14:25:56,516 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-23T14:25:56,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:56,517 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2902): Flushing a36a6f46a23b82a88d429516318dcd05 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-23T14:25:56,528 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a36a6f46a23b82a88d429516318dcd05/B of a36a6f46a23b82a88d429516318dcd05 into febef176354b465fb8683f9a261d65fe(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:25:56,528 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a36a6f46a23b82a88d429516318dcd05: 2024-11-23T14:25:56,528 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., storeName=a36a6f46a23b82a88d429516318dcd05/B, priority=13, startTime=1732371956439; duration=0sec 2024-11-23T14:25:56,528 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:25:56,528 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a36a6f46a23b82a88d429516318dcd05:B 2024-11-23T14:25:56,529 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:25:56,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742049_1225 (size=4469) 2024-11-23T14:25:56,530 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a36a6f46a23b82a88d429516318dcd05#A#compaction#190 average throughput is 0.43 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:25:56,531 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:25:56,531 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/A/f5bd8bce993f4af89e9533e7db48d9e1 is 175, key is test_row_0/A:col10/1732371953923/Put/seqid=0 2024-11-23T14:25:56,531 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1541): a36a6f46a23b82a88d429516318dcd05/C is initiating minor compaction (all files) 2024-11-23T14:25:56,531 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a36a6f46a23b82a88d429516318dcd05/C in TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:56,531 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/2cf16f5675ea441286131274963dbf8e, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/aa54469a03084c8da913c2a8eadbf2aa, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/39508c3260854100a9713cdefcbbd5b9] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp, totalSize=36.4 K 2024-11-23T14:25:56,532 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 2cf16f5675ea441286131274963dbf8e, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1732371951520 2024-11-23T14:25:56,535 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting aa54469a03084c8da913c2a8eadbf2aa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1732371952719 2024-11-23T14:25:56,536 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 39508c3260854100a9713cdefcbbd5b9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732371953923 2024-11-23T14:25:56,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411236415966190b94bd6913939fb5d32503b_a36a6f46a23b82a88d429516318dcd05 is 50, key is test_row_0/A:col10/1732371955169/Put/seqid=0 2024-11-23T14:25:56,571 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a36a6f46a23b82a88d429516318dcd05#C#compaction#192 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:25:56,572 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/C/27a510ce5bcc4d56b795a29ae08f27a5 is 50, key is test_row_0/C:col10/1732371953923/Put/seqid=0 2024-11-23T14:25:56,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742050_1226 (size=31903) 2024-11-23T14:25:56,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742051_1227 (size=12454) 2024-11-23T14:25:56,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:56,660 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411236415966190b94bd6913939fb5d32503b_a36a6f46a23b82a88d429516318dcd05 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411236415966190b94bd6913939fb5d32503b_a36a6f46a23b82a88d429516318dcd05 2024-11-23T14:25:56,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/A/9ea6a4a8244e4d3098d41da979be4759, store: [table=TestAcidGuarantees family=A region=a36a6f46a23b82a88d429516318dcd05] 2024-11-23T14:25:56,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/A/9ea6a4a8244e4d3098d41da979be4759 is 175, key is test_row_0/A:col10/1732371955169/Put/seqid=0 2024-11-23T14:25:56,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742052_1228 (size=12949) 2024-11-23T14:25:56,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742053_1229 (size=31255) 2024-11-23T14:25:56,707 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=311, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/A/9ea6a4a8244e4d3098d41da979be4759 2024-11-23T14:25:56,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/B/faca79b92da84a35a82d45cccf8c91e4 is 50, key is test_row_0/B:col10/1732371955169/Put/seqid=0 2024-11-23T14:25:56,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742054_1230 (size=12301) 2024-11-23T14:25:56,779 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/B/faca79b92da84a35a82d45cccf8c91e4 2024-11-23T14:25:56,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/C/f215aaf8d290437ba61fd5c4e7249c4f is 50, key is test_row_0/C:col10/1732371955169/Put/seqid=0 2024-11-23T14:25:56,825 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. as already flushing 2024-11-23T14:25:56,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on a36a6f46a23b82a88d429516318dcd05 2024-11-23T14:25:56,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742055_1231 (size=12301) 2024-11-23T14:25:56,841 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/C/f215aaf8d290437ba61fd5c4e7249c4f 2024-11-23T14:25:56,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/A/9ea6a4a8244e4d3098d41da979be4759 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/9ea6a4a8244e4d3098d41da979be4759 2024-11-23T14:25:56,865 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/9ea6a4a8244e4d3098d41da979be4759, entries=150, sequenceid=311, filesize=30.5 K 2024-11-23T14:25:56,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/B/faca79b92da84a35a82d45cccf8c91e4 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/faca79b92da84a35a82d45cccf8c91e4 2024-11-23T14:25:56,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:56,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40478 deadline: 1732372016868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:56,873 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:56,874 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:56,874 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:56,875 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/faca79b92da84a35a82d45cccf8c91e4, entries=150, sequenceid=311, filesize=12.0 K 2024-11-23T14:25:56,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/C/f215aaf8d290437ba61fd5c4e7249c4f as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/f215aaf8d290437ba61fd5c4e7249c4f 2024-11-23T14:25:56,881 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:56,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40464 deadline: 1732372016880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:56,882 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to c6f7b1897123,35385,1732371902767 for 'test_row_2' in a36a6f46a23b82a88d429516318dcd05 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18270 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-23T14:25:56,882 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:56,882 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:56,882 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:56,896 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/f215aaf8d290437ba61fd5c4e7249c4f, entries=150, sequenceid=311, filesize=12.0 K 2024-11-23T14:25:56,898 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for a36a6f46a23b82a88d429516318dcd05 in 381ms, sequenceid=311, compaction requested=false 2024-11-23T14:25:56,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for a36a6f46a23b82a88d429516318dcd05: 2024-11-23T14:25:56,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:56,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-11-23T14:25:56,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-11-23T14:25:56,902 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-23T14:25:56,902 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7620 sec 2024-11-23T14:25:56,904 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T14:25:56,904 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a36a6f46a23b82a88d429516318dcd05 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-23T14:25:56,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on a36a6f46a23b82a88d429516318dcd05 2024-11-23T14:25:56,905 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees in 1.7710 sec 2024-11-23T14:25:56,918 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123ddc61febff8c4335a162f0f4d5c3a91f_a36a6f46a23b82a88d429516318dcd05 is 50, key is test_row_0/A:col10/1732371956903/Put/seqid=0 2024-11-23T14:25:56,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742056_1232 (size=14994) 2024-11-23T14:25:56,936 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:56,942 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123ddc61febff8c4335a162f0f4d5c3a91f_a36a6f46a23b82a88d429516318dcd05 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123ddc61febff8c4335a162f0f4d5c3a91f_a36a6f46a23b82a88d429516318dcd05 2024-11-23T14:25:56,943 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/A/1bf4f03a0a8d4fb1a093e0396e9766f1, store: [table=TestAcidGuarantees family=A region=a36a6f46a23b82a88d429516318dcd05] 2024-11-23T14:25:56,944 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/A/1bf4f03a0a8d4fb1a093e0396e9766f1 is 175, key is test_row_0/A:col10/1732371956903/Put/seqid=0 2024-11-23T14:25:56,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742057_1233 (size=39949) 2024-11-23T14:25:56,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:56,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40436 deadline: 1732372016970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:56,971 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:56,971 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:56,971 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:56,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:56,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40438 deadline: 1732372016971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:56,974 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:56,974 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:56,974 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:56,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:56,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40478 deadline: 1732372016980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:56,983 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:56,983 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:56,983 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:57,017 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/A/f5bd8bce993f4af89e9533e7db48d9e1 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/f5bd8bce993f4af89e9533e7db48d9e1 2024-11-23T14:25:57,022 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a36a6f46a23b82a88d429516318dcd05/A of a36a6f46a23b82a88d429516318dcd05 into f5bd8bce993f4af89e9533e7db48d9e1(size=31.2 K), total size for store is 61.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:25:57,022 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a36a6f46a23b82a88d429516318dcd05: 2024-11-23T14:25:57,022 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., storeName=a36a6f46a23b82a88d429516318dcd05/A, priority=13, startTime=1732371956439; duration=0sec 2024-11-23T14:25:57,022 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:25:57,022 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a36a6f46a23b82a88d429516318dcd05:A 2024-11-23T14:25:57,072 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/C/27a510ce5bcc4d56b795a29ae08f27a5 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/27a510ce5bcc4d56b795a29ae08f27a5 2024-11-23T14:25:57,077 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a36a6f46a23b82a88d429516318dcd05/C of a36a6f46a23b82a88d429516318dcd05 into 27a510ce5bcc4d56b795a29ae08f27a5(size=12.6 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:25:57,077 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a36a6f46a23b82a88d429516318dcd05: 2024-11-23T14:25:57,077 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., storeName=a36a6f46a23b82a88d429516318dcd05/C, priority=13, startTime=1732371956439; duration=0sec 2024-11-23T14:25:57,077 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:25:57,077 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a36a6f46a23b82a88d429516318dcd05:C 2024-11-23T14:25:57,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:57,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40436 deadline: 1732372017080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:57,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:57,084 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:57,084 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:57,084 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:57,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40438 deadline: 1732372017080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:57,084 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:57,084 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:57,084 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:57,192 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:57,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40478 deadline: 1732372017190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:57,192 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:57,192 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:57,192 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:57,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-23T14:25:57,291 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T14:25:57,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:57,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40436 deadline: 1732372017290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:57,292 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:57,292 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:57,292 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:57,292 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:57,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40438 deadline: 1732372017290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:57,292 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-23T14:25:57,293 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:57,293 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:57,293 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:57,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees 2024-11-23T14:25:57,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-23T14:25:57,298 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T14:25:57,299 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T14:25:57,299 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T14:25:57,330 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:57,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 268 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40476 deadline: 1732372017330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:57,331 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to c6f7b1897123,35385,1732371902767 for 'test_row_2' in a36a6f46a23b82a88d429516318dcd05 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2153 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-23T14:25:57,332 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:57,332 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:57,332 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:57,351 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=325, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/A/1bf4f03a0a8d4fb1a093e0396e9766f1 2024-11-23T14:25:57,359 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/B/c63bba85f2604c93b7e8d63cd17ab4a5 is 50, key is test_row_0/B:col10/1732371956903/Put/seqid=0 2024-11-23T14:25:57,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742058_1234 (size=12301) 2024-11-23T14:25:57,380 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/B/c63bba85f2604c93b7e8d63cd17ab4a5 2024-11-23T14:25:57,389 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/C/974ee6d90dab4bcda2d636bc8471cad4 is 50, key is test_row_0/C:col10/1732371956903/Put/seqid=0 2024-11-23T14:25:57,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742059_1235 (size=12301) 2024-11-23T14:25:57,399 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/C/974ee6d90dab4bcda2d636bc8471cad4 2024-11-23T14:25:57,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-23T14:25:57,406 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/A/1bf4f03a0a8d4fb1a093e0396e9766f1 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/1bf4f03a0a8d4fb1a093e0396e9766f1 2024-11-23T14:25:57,413 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/1bf4f03a0a8d4fb1a093e0396e9766f1, entries=200, sequenceid=325, filesize=39.0 K 2024-11-23T14:25:57,415 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/B/c63bba85f2604c93b7e8d63cd17ab4a5 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/c63bba85f2604c93b7e8d63cd17ab4a5 2024-11-23T14:25:57,419 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/c63bba85f2604c93b7e8d63cd17ab4a5, entries=150, sequenceid=325, filesize=12.0 K 2024-11-23T14:25:57,421 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/C/974ee6d90dab4bcda2d636bc8471cad4 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/974ee6d90dab4bcda2d636bc8471cad4 2024-11-23T14:25:57,429 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/974ee6d90dab4bcda2d636bc8471cad4, entries=150, sequenceid=325, filesize=12.0 K 2024-11-23T14:25:57,430 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for a36a6f46a23b82a88d429516318dcd05 in 525ms, sequenceid=325, compaction requested=true 2024-11-23T14:25:57,430 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a36a6f46a23b82a88d429516318dcd05: 2024-11-23T14:25:57,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a36a6f46a23b82a88d429516318dcd05:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T14:25:57,430 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:25:57,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:25:57,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a36a6f46a23b82a88d429516318dcd05:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T14:25:57,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:25:57,430 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:25:57,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a36a6f46a23b82a88d429516318dcd05:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T14:25:57,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:25:57,434 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:25:57,434 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1541): a36a6f46a23b82a88d429516318dcd05/B is initiating minor compaction (all files) 2024-11-23T14:25:57,434 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a36a6f46a23b82a88d429516318dcd05/B in TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:57,434 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/febef176354b465fb8683f9a261d65fe, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/faca79b92da84a35a82d45cccf8c91e4, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/c63bba85f2604c93b7e8d63cd17ab4a5] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp, totalSize=36.7 K 2024-11-23T14:25:57,435 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103107 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:25:57,435 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1541): a36a6f46a23b82a88d429516318dcd05/A is initiating minor compaction (all files) 2024-11-23T14:25:57,435 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a36a6f46a23b82a88d429516318dcd05/A in TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:57,435 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/f5bd8bce993f4af89e9533e7db48d9e1, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/9ea6a4a8244e4d3098d41da979be4759, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/1bf4f03a0a8d4fb1a093e0396e9766f1] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp, totalSize=100.7 K 2024-11-23T14:25:57,435 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:57,435 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. files: [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/f5bd8bce993f4af89e9533e7db48d9e1, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/9ea6a4a8244e4d3098d41da979be4759, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/1bf4f03a0a8d4fb1a093e0396e9766f1] 2024-11-23T14:25:57,436 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting febef176354b465fb8683f9a261d65fe, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732371953923 2024-11-23T14:25:57,436 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting f5bd8bce993f4af89e9533e7db48d9e1, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732371953923 2024-11-23T14:25:57,437 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting faca79b92da84a35a82d45cccf8c91e4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1732371955164 2024-11-23T14:25:57,437 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ea6a4a8244e4d3098d41da979be4759, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1732371955164 2024-11-23T14:25:57,437 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1bf4f03a0a8d4fb1a093e0396e9766f1, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1732371956852 2024-11-23T14:25:57,437 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting c63bba85f2604c93b7e8d63cd17ab4a5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1732371956852 2024-11-23T14:25:57,455 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-23T14:25:57,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:57,456 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2902): Flushing a36a6f46a23b82a88d429516318dcd05 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-23T14:25:57,462 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a36a6f46a23b82a88d429516318dcd05#B#compaction#198 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:25:57,463 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/B/f56b2fee6f024515be8d16bc377d5882 is 50, key is test_row_0/B:col10/1732371956903/Put/seqid=0 2024-11-23T14:25:57,464 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a36a6f46a23b82a88d429516318dcd05] 2024-11-23T14:25:57,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112349f8bad21ef242d9bf1213f896f3c73f_a36a6f46a23b82a88d429516318dcd05 is 50, key is test_row_0/A:col10/1732371956957/Put/seqid=0 2024-11-23T14:25:57,475 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241123cabd5696b43f49e28de221c861fd00fd_a36a6f46a23b82a88d429516318dcd05 store=[table=TestAcidGuarantees family=A region=a36a6f46a23b82a88d429516318dcd05] 2024-11-23T14:25:57,476 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241123cabd5696b43f49e28de221c861fd00fd_a36a6f46a23b82a88d429516318dcd05, store=[table=TestAcidGuarantees family=A region=a36a6f46a23b82a88d429516318dcd05] 2024-11-23T14:25:57,476 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123cabd5696b43f49e28de221c861fd00fd_a36a6f46a23b82a88d429516318dcd05 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a36a6f46a23b82a88d429516318dcd05] 2024-11-23T14:25:57,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742060_1236 (size=13051) 2024-11-23T14:25:57,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on a36a6f46a23b82a88d429516318dcd05 2024-11-23T14:25:57,503 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. as already flushing 2024-11-23T14:25:57,544 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:57,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40478 deadline: 1732372017540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:57,544 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:57,544 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:57,544 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:57,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742061_1237 (size=12454) 2024-11-23T14:25:57,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:57,550 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112349f8bad21ef242d9bf1213f896f3c73f_a36a6f46a23b82a88d429516318dcd05 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112349f8bad21ef242d9bf1213f896f3c73f_a36a6f46a23b82a88d429516318dcd05 2024-11-23T14:25:57,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/A/d02ca8bb91d64dc3ae532a936cb6a83f, store: [table=TestAcidGuarantees family=A region=a36a6f46a23b82a88d429516318dcd05] 2024-11-23T14:25:57,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/A/d02ca8bb91d64dc3ae532a936cb6a83f is 175, key is test_row_0/A:col10/1732371956957/Put/seqid=0 2024-11-23T14:25:57,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742062_1238 (size=4469) 2024-11-23T14:25:57,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742063_1239 (size=31255) 2024-11-23T14:25:57,577 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=350, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/A/d02ca8bb91d64dc3ae532a936cb6a83f 2024-11-23T14:25:57,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/B/703a76eb324f4cfd83e6672815371ce7 is 50, key is test_row_0/B:col10/1732371956957/Put/seqid=0 2024-11-23T14:25:57,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742064_1240 (size=12301) 2024-11-23T14:25:57,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:57,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40436 deadline: 1732372017600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:57,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:57,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40438 deadline: 1732372017600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:57,601 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:57,601 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:57,601 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:57,608 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:57,608 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:57,608 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:57,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-23T14:25:57,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:57,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40478 deadline: 1732372017650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:57,657 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:57,657 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:57,657 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:57,863 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:57,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40478 deadline: 1732372017860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:57,863 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:57,863 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:57,863 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:57,885 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/B/f56b2fee6f024515be8d16bc377d5882 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/f56b2fee6f024515be8d16bc377d5882 2024-11-23T14:25:57,890 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a36a6f46a23b82a88d429516318dcd05/B of a36a6f46a23b82a88d429516318dcd05 into f56b2fee6f024515be8d16bc377d5882(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:25:57,890 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a36a6f46a23b82a88d429516318dcd05: 2024-11-23T14:25:57,891 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., storeName=a36a6f46a23b82a88d429516318dcd05/B, priority=13, startTime=1732371957430; duration=0sec 2024-11-23T14:25:57,891 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:25:57,891 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a36a6f46a23b82a88d429516318dcd05:B 2024-11-23T14:25:57,891 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:25:57,892 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:25:57,892 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1541): a36a6f46a23b82a88d429516318dcd05/C is initiating minor compaction (all files) 2024-11-23T14:25:57,892 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a36a6f46a23b82a88d429516318dcd05/C in TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:57,892 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/27a510ce5bcc4d56b795a29ae08f27a5, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/f215aaf8d290437ba61fd5c4e7249c4f, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/974ee6d90dab4bcda2d636bc8471cad4] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp, totalSize=36.7 K 2024-11-23T14:25:57,892 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 27a510ce5bcc4d56b795a29ae08f27a5, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732371953923 2024-11-23T14:25:57,893 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting f215aaf8d290437ba61fd5c4e7249c4f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1732371955164 2024-11-23T14:25:57,893 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 974ee6d90dab4bcda2d636bc8471cad4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1732371956852 2024-11-23T14:25:57,901 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a36a6f46a23b82a88d429516318dcd05#C#compaction#202 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:25:57,902 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/C/6734130028634709a455968f8595878a is 50, key is test_row_0/C:col10/1732371956903/Put/seqid=0 2024-11-23T14:25:57,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742065_1241 (size=13051) 2024-11-23T14:25:57,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-23T14:25:57,962 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a36a6f46a23b82a88d429516318dcd05#A#compaction#199 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:25:57,963 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/A/244676194bd945fc998aca2f6c3d301f is 175, key is test_row_0/A:col10/1732371956903/Put/seqid=0 2024-11-23T14:25:57,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742066_1242 (size=32005) 2024-11-23T14:25:58,001 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/B/703a76eb324f4cfd83e6672815371ce7 2024-11-23T14:25:58,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/C/8eb224d706284900b84cd38799da16c0 is 50, key is test_row_0/C:col10/1732371956957/Put/seqid=0 2024-11-23T14:25:58,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742067_1243 (size=12301) 2024-11-23T14:25:58,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:58,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40436 deadline: 1732372018109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:58,112 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:58,112 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:58,112 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:58,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:58,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40438 deadline: 1732372018120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:58,121 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:58,121 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:58,121 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:58,165 INFO [Thread-730 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-730. 2024-11-23T14:25:58,165 DEBUG [Thread-730 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T14:25:58,165 DEBUG [Thread-730 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:25:58,165 DEBUG [Thread-730 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:25:58,165 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T14:25:58,167 INFO [Thread-732 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-732. 2024-11-23T14:25:58,167 DEBUG [Thread-732 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T14:25:58,167 DEBUG [Thread-732 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:25:58,167 DEBUG [Thread-732 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:25:58,167 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T14:25:58,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:25:58,170 INFO [Thread-736 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-736. 2024-11-23T14:25:58,170 DEBUG [Thread-736 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T14:25:58,170 DEBUG [Thread-736 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:25:58,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40478 deadline: 1732372018170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:25:58,171 INFO [Thread-734 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-734. 2024-11-23T14:25:58,171 DEBUG [Thread-736 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:25:58,171 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T14:25:58,171 DEBUG [Thread-734 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T14:25:58,171 DEBUG [Thread-734 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:25:58,171 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:58,171 DEBUG [Thread-734 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:25:58,171 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:25:58,171 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T14:25:58,171 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:25:58,312 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/C/6734130028634709a455968f8595878a as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/6734130028634709a455968f8595878a 2024-11-23T14:25:58,316 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a36a6f46a23b82a88d429516318dcd05/C of a36a6f46a23b82a88d429516318dcd05 into 6734130028634709a455968f8595878a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:25:58,316 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a36a6f46a23b82a88d429516318dcd05: 2024-11-23T14:25:58,316 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., storeName=a36a6f46a23b82a88d429516318dcd05/C, priority=13, startTime=1732371957430; duration=0sec 2024-11-23T14:25:58,316 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:25:58,317 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a36a6f46a23b82a88d429516318dcd05:C 2024-11-23T14:25:58,379 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/A/244676194bd945fc998aca2f6c3d301f as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/244676194bd945fc998aca2f6c3d301f 2024-11-23T14:25:58,384 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a36a6f46a23b82a88d429516318dcd05/A of a36a6f46a23b82a88d429516318dcd05 into 244676194bd945fc998aca2f6c3d301f(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:25:58,384 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a36a6f46a23b82a88d429516318dcd05: 2024-11-23T14:25:58,384 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., storeName=a36a6f46a23b82a88d429516318dcd05/A, priority=13, startTime=1732371957430; duration=0sec 2024-11-23T14:25:58,384 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:25:58,384 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a36a6f46a23b82a88d429516318dcd05:A 2024-11-23T14:25:58,427 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/C/8eb224d706284900b84cd38799da16c0 2024-11-23T14:25:58,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-23T14:25:58,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/A/d02ca8bb91d64dc3ae532a936cb6a83f as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/d02ca8bb91d64dc3ae532a936cb6a83f 2024-11-23T14:25:58,438 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/d02ca8bb91d64dc3ae532a936cb6a83f, entries=150, sequenceid=350, filesize=30.5 K 2024-11-23T14:25:58,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/B/703a76eb324f4cfd83e6672815371ce7 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/703a76eb324f4cfd83e6672815371ce7 2024-11-23T14:25:58,443 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/703a76eb324f4cfd83e6672815371ce7, entries=150, sequenceid=350, filesize=12.0 K 2024-11-23T14:25:58,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/C/8eb224d706284900b84cd38799da16c0 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/8eb224d706284900b84cd38799da16c0 2024-11-23T14:25:58,448 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/8eb224d706284900b84cd38799da16c0, entries=150, sequenceid=350, filesize=12.0 K 2024-11-23T14:25:58,449 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for a36a6f46a23b82a88d429516318dcd05 in 994ms, sequenceid=350, compaction requested=false 2024-11-23T14:25:58,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2603): Flush status journal for a36a6f46a23b82a88d429516318dcd05: 2024-11-23T14:25:58,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:25:58,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-11-23T14:25:58,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-11-23T14:25:58,453 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-23T14:25:58,453 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1520 sec 2024-11-23T14:25:58,455 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees in 1.1620 sec 2024-11-23T14:25:58,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on a36a6f46a23b82a88d429516318dcd05 2024-11-23T14:25:58,683 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T14:25:58,683 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a36a6f46a23b82a88d429516318dcd05 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-23T14:25:58,684 INFO [Thread-721 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-721. 2024-11-23T14:25:58,684 DEBUG [Thread-721 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T14:25:58,684 DEBUG [Thread-721 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:25:58,684 DEBUG [Thread-721 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:25:58,684 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T14:25:58,695 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123a092e3e1248a4953bfae05fcf3a37064_a36a6f46a23b82a88d429516318dcd05 is 50, key is test_row_0/A:col10/1732371957531/Put/seqid=0 2024-11-23T14:25:58,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742068_1244 (size=12454) 2024-11-23T14:25:59,099 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:25:59,103 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123a092e3e1248a4953bfae05fcf3a37064_a36a6f46a23b82a88d429516318dcd05 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123a092e3e1248a4953bfae05fcf3a37064_a36a6f46a23b82a88d429516318dcd05 2024-11-23T14:25:59,104 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/A/dc2468669dbf4192b6dfb48156577419, store: [table=TestAcidGuarantees family=A region=a36a6f46a23b82a88d429516318dcd05] 2024-11-23T14:25:59,105 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/A/dc2468669dbf4192b6dfb48156577419 is 175, key is test_row_0/A:col10/1732371957531/Put/seqid=0 2024-11-23T14:25:59,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742069_1245 (size=31255) 2024-11-23T14:25:59,132 INFO [Thread-719 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-719. 2024-11-23T14:25:59,132 INFO [Thread-723 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-723. 2024-11-23T14:25:59,132 DEBUG [Thread-719 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T14:25:59,132 DEBUG [Thread-723 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T14:25:59,132 DEBUG [Thread-719 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:25:59,132 DEBUG [Thread-723 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:25:59,132 DEBUG [Thread-719 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:25:59,132 DEBUG [Thread-723 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:25:59,132 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T14:25:59,132 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T14:25:59,352 INFO [Thread-725 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-725. 2024-11-23T14:25:59,352 DEBUG [Thread-725 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T14:25:59,352 DEBUG [Thread-725 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:25:59,352 DEBUG [Thread-725 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:25:59,352 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T14:25:59,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-23T14:25:59,441 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T14:25:59,509 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=367, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/A/dc2468669dbf4192b6dfb48156577419 2024-11-23T14:25:59,518 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/B/35433cb7fb6f4c538a5ef5e999dc4ddd is 50, key is test_row_0/B:col10/1732371957531/Put/seqid=0 2024-11-23T14:25:59,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742070_1246 (size=12301) 2024-11-23T14:25:59,924 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/B/35433cb7fb6f4c538a5ef5e999dc4ddd 2024-11-23T14:25:59,938 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/C/f166712c952f4839bbd631f5fc29ef5c is 50, key is test_row_0/C:col10/1732371957531/Put/seqid=0 2024-11-23T14:25:59,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742071_1247 (size=12301) 2024-11-23T14:26:00,343 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/C/f166712c952f4839bbd631f5fc29ef5c 2024-11-23T14:26:00,354 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/A/dc2468669dbf4192b6dfb48156577419 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/dc2468669dbf4192b6dfb48156577419 2024-11-23T14:26:00,359 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/dc2468669dbf4192b6dfb48156577419, entries=150, sequenceid=367, filesize=30.5 K 2024-11-23T14:26:00,360 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/B/35433cb7fb6f4c538a5ef5e999dc4ddd as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/35433cb7fb6f4c538a5ef5e999dc4ddd 2024-11-23T14:26:00,365 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/35433cb7fb6f4c538a5ef5e999dc4ddd, entries=150, sequenceid=367, filesize=12.0 K 2024-11-23T14:26:00,367 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/C/f166712c952f4839bbd631f5fc29ef5c as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/f166712c952f4839bbd631f5fc29ef5c 2024-11-23T14:26:00,372 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/f166712c952f4839bbd631f5fc29ef5c, entries=150, sequenceid=367, filesize=12.0 K 2024-11-23T14:26:00,373 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=20.13 KB/20610 for a36a6f46a23b82a88d429516318dcd05 in 1690ms, sequenceid=367, compaction requested=true 2024-11-23T14:26:00,373 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a36a6f46a23b82a88d429516318dcd05: 2024-11-23T14:26:00,373 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a36a6f46a23b82a88d429516318dcd05:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T14:26:00,373 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:26:00,373 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a36a6f46a23b82a88d429516318dcd05:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T14:26:00,373 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:26:00,373 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:26:00,373 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:26:00,373 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a36a6f46a23b82a88d429516318dcd05:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T14:26:00,373 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:26:00,374 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94515 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:26:00,374 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:26:00,374 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1541): a36a6f46a23b82a88d429516318dcd05/A is initiating minor compaction (all files) 2024-11-23T14:26:00,374 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1541): a36a6f46a23b82a88d429516318dcd05/B is initiating minor compaction (all files) 2024-11-23T14:26:00,375 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a36a6f46a23b82a88d429516318dcd05/A in TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:26:00,375 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a36a6f46a23b82a88d429516318dcd05/B in TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:26:00,375 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/244676194bd945fc998aca2f6c3d301f, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/d02ca8bb91d64dc3ae532a936cb6a83f, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/dc2468669dbf4192b6dfb48156577419] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp, totalSize=92.3 K 2024-11-23T14:26:00,375 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/f56b2fee6f024515be8d16bc377d5882, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/703a76eb324f4cfd83e6672815371ce7, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/35433cb7fb6f4c538a5ef5e999dc4ddd] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp, totalSize=36.8 K 2024-11-23T14:26:00,375 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:26:00,375 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. files: [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/244676194bd945fc998aca2f6c3d301f, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/d02ca8bb91d64dc3ae532a936cb6a83f, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/dc2468669dbf4192b6dfb48156577419] 2024-11-23T14:26:00,375 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting f56b2fee6f024515be8d16bc377d5882, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1732371956852 2024-11-23T14:26:00,375 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 244676194bd945fc998aca2f6c3d301f, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1732371956852 2024-11-23T14:26:00,376 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 703a76eb324f4cfd83e6672815371ce7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1732371956957 2024-11-23T14:26:00,376 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting d02ca8bb91d64dc3ae532a936cb6a83f, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1732371956957 2024-11-23T14:26:00,376 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 35433cb7fb6f4c538a5ef5e999dc4ddd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1732371957506 2024-11-23T14:26:00,376 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc2468669dbf4192b6dfb48156577419, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1732371957506 2024-11-23T14:26:00,386 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a36a6f46a23b82a88d429516318dcd05] 2024-11-23T14:26:00,388 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a36a6f46a23b82a88d429516318dcd05#B#compaction#207 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:26:00,389 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/B/a7d919935b174740a05f91f38ea12021 is 50, key is test_row_0/B:col10/1732371957531/Put/seqid=0 2024-11-23T14:26:00,390 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241123423af1eb533844cc99a32eb135d14d1a_a36a6f46a23b82a88d429516318dcd05 store=[table=TestAcidGuarantees family=A region=a36a6f46a23b82a88d429516318dcd05] 2024-11-23T14:26:00,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742072_1248 (size=13153) 2024-11-23T14:26:00,416 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241123423af1eb533844cc99a32eb135d14d1a_a36a6f46a23b82a88d429516318dcd05, store=[table=TestAcidGuarantees family=A region=a36a6f46a23b82a88d429516318dcd05] 2024-11-23T14:26:00,416 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123423af1eb533844cc99a32eb135d14d1a_a36a6f46a23b82a88d429516318dcd05 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a36a6f46a23b82a88d429516318dcd05] 2024-11-23T14:26:00,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742073_1249 (size=4469) 2024-11-23T14:26:00,797 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-23T14:26:00,803 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/B/a7d919935b174740a05f91f38ea12021 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/a7d919935b174740a05f91f38ea12021 2024-11-23T14:26:00,808 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a36a6f46a23b82a88d429516318dcd05/B of a36a6f46a23b82a88d429516318dcd05 into a7d919935b174740a05f91f38ea12021(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:26:00,808 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a36a6f46a23b82a88d429516318dcd05: 2024-11-23T14:26:00,808 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., storeName=a36a6f46a23b82a88d429516318dcd05/B, priority=13, startTime=1732371960373; duration=0sec 2024-11-23T14:26:00,809 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:26:00,809 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a36a6f46a23b82a88d429516318dcd05:B 2024-11-23T14:26:00,809 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:26:00,810 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:26:00,810 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1541): a36a6f46a23b82a88d429516318dcd05/C is initiating minor compaction (all files) 2024-11-23T14:26:00,810 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a36a6f46a23b82a88d429516318dcd05/C in TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:26:00,810 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/6734130028634709a455968f8595878a, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/8eb224d706284900b84cd38799da16c0, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/f166712c952f4839bbd631f5fc29ef5c] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp, totalSize=36.8 K 2024-11-23T14:26:00,811 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 6734130028634709a455968f8595878a, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1732371956852 2024-11-23T14:26:00,811 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 8eb224d706284900b84cd38799da16c0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1732371956957 2024-11-23T14:26:00,812 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting f166712c952f4839bbd631f5fc29ef5c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1732371957506 2024-11-23T14:26:00,821 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a36a6f46a23b82a88d429516318dcd05#A#compaction#208 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:26:00,821 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/A/0e37af91b0134712940f875ed8d3237b is 175, key is test_row_0/A:col10/1732371957531/Put/seqid=0 2024-11-23T14:26:00,822 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a36a6f46a23b82a88d429516318dcd05#C#compaction#209 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:26:00,823 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/C/d4061db8a38e4e4b9c89d01c51590ba0 is 50, key is test_row_0/C:col10/1732371957531/Put/seqid=0 2024-11-23T14:26:00,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742074_1250 (size=32107) 2024-11-23T14:26:00,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742075_1251 (size=13153) 2024-11-23T14:26:01,232 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/A/0e37af91b0134712940f875ed8d3237b as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/0e37af91b0134712940f875ed8d3237b 2024-11-23T14:26:01,238 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/C/d4061db8a38e4e4b9c89d01c51590ba0 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/d4061db8a38e4e4b9c89d01c51590ba0 2024-11-23T14:26:01,238 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a36a6f46a23b82a88d429516318dcd05/A of a36a6f46a23b82a88d429516318dcd05 into 0e37af91b0134712940f875ed8d3237b(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:26:01,238 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a36a6f46a23b82a88d429516318dcd05: 2024-11-23T14:26:01,238 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., storeName=a36a6f46a23b82a88d429516318dcd05/A, priority=13, startTime=1732371960373; duration=0sec 2024-11-23T14:26:01,239 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:26:01,239 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a36a6f46a23b82a88d429516318dcd05:A 2024-11-23T14:26:01,249 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a36a6f46a23b82a88d429516318dcd05/C of a36a6f46a23b82a88d429516318dcd05 into d4061db8a38e4e4b9c89d01c51590ba0(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:26:01,249 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a36a6f46a23b82a88d429516318dcd05: 2024-11-23T14:26:01,249 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., storeName=a36a6f46a23b82a88d429516318dcd05/C, priority=13, startTime=1732371960373; duration=0sec 2024-11-23T14:26:01,249 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:26:01,249 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a36a6f46a23b82a88d429516318dcd05:C 2024-11-23T14:26:06,955 INFO [Thread-727 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-727. 2024-11-23T14:26:06,955 DEBUG [Thread-727 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T14:26:06,956 DEBUG [Thread-727 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:26:06,956 DEBUG [Thread-727 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:26:06,956 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T14:26:06,957 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-23T14:26:06,957 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 24 2024-11-23T14:26:06,957 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 29 2024-11-23T14:26:06,957 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 15 2024-11-23T14:26:06,957 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 215 2024-11-23T14:26:06,957 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 9 2024-11-23T14:26:06,957 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-23T14:26:06,957 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4112 2024-11-23T14:26:06,957 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4131 2024-11-23T14:26:06,957 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-23T14:26:06,957 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1769 2024-11-23T14:26:06,957 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5303 rows 2024-11-23T14:26:06,957 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1763 2024-11-23T14:26:06,958 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5286 rows 2024-11-23T14:26:06,958 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-23T14:26:06,958 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T14:26:06,958 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:26:06,959 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:26:06,959 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T14:26:06,961 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-23T14:26:06,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-23T14:26:06,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-23T14:26:06,965 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732371966964"}]},"ts":"1732371966964"} 2024-11-23T14:26:06,980 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-23T14:26:06,980 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-23T14:26:06,981 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-23T14:26:06,983 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a36a6f46a23b82a88d429516318dcd05, UNASSIGN}] 2024-11-23T14:26:06,983 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a36a6f46a23b82a88d429516318dcd05, UNASSIGN 2024-11-23T14:26:06,984 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=a36a6f46a23b82a88d429516318dcd05, regionState=CLOSING, regionLocation=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:06,985 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a36a6f46a23b82a88d429516318dcd05, UNASSIGN because future has completed 2024-11-23T14:26:06,985 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-23T14:26:06,985 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; CloseRegionProcedure a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767}] 2024-11-23T14:26:07,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-23T14:26:07,139 INFO [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] handler.UnassignRegionHandler(122): Close a36a6f46a23b82a88d429516318dcd05 2024-11-23T14:26:07,139 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-23T14:26:07,139 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1722): Closing a36a6f46a23b82a88d429516318dcd05, disabling compactions & flushes 2024-11-23T14:26:07,139 INFO [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:26:07,139 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:26:07,139 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. after waiting 0 ms 2024-11-23T14:26:07,139 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:26:07,140 INFO [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(2902): Flushing a36a6f46a23b82a88d429516318dcd05 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-23T14:26:07,152 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112361b9b2f6f9d345fd89c97507d3c3d6e9_a36a6f46a23b82a88d429516318dcd05 is 50, key is test_row_0/A:col10/1732371959131/Put/seqid=0 2024-11-23T14:26:07,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742076_1252 (size=12454) 2024-11-23T14:26:07,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-23T14:26:07,558 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:26:07,568 INFO [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112361b9b2f6f9d345fd89c97507d3c3d6e9_a36a6f46a23b82a88d429516318dcd05 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112361b9b2f6f9d345fd89c97507d3c3d6e9_a36a6f46a23b82a88d429516318dcd05 2024-11-23T14:26:07,570 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/A/229ebd154cad49d493832d4c7355b1ac, store: [table=TestAcidGuarantees family=A region=a36a6f46a23b82a88d429516318dcd05] 2024-11-23T14:26:07,570 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/A/229ebd154cad49d493832d4c7355b1ac is 175, key is test_row_0/A:col10/1732371959131/Put/seqid=0 2024-11-23T14:26:07,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742077_1253 (size=31255) 2024-11-23T14:26:07,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-23T14:26:07,975 INFO [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=377, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/A/229ebd154cad49d493832d4c7355b1ac 2024-11-23T14:26:07,985 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/B/12fa08965eb94d5e827d55e30f55646a is 50, key is test_row_0/B:col10/1732371959131/Put/seqid=0 2024-11-23T14:26:07,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742078_1254 (size=12301) 2024-11-23T14:26:08,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-23T14:26:08,392 INFO [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=377 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/B/12fa08965eb94d5e827d55e30f55646a 2024-11-23T14:26:08,406 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/C/3add8bc178b5494bbf8f236489c261e5 is 50, key is test_row_0/C:col10/1732371959131/Put/seqid=0 2024-11-23T14:26:08,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742079_1255 (size=12301) 2024-11-23T14:26:08,811 INFO [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=377 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/C/3add8bc178b5494bbf8f236489c261e5 2024-11-23T14:26:08,821 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/A/229ebd154cad49d493832d4c7355b1ac as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/229ebd154cad49d493832d4c7355b1ac 2024-11-23T14:26:08,826 INFO [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/229ebd154cad49d493832d4c7355b1ac, entries=150, sequenceid=377, filesize=30.5 K 2024-11-23T14:26:08,827 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/B/12fa08965eb94d5e827d55e30f55646a as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/12fa08965eb94d5e827d55e30f55646a 2024-11-23T14:26:08,831 INFO [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/12fa08965eb94d5e827d55e30f55646a, entries=150, sequenceid=377, filesize=12.0 K 2024-11-23T14:26:08,832 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/.tmp/C/3add8bc178b5494bbf8f236489c261e5 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/3add8bc178b5494bbf8f236489c261e5 2024-11-23T14:26:08,837 INFO [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/3add8bc178b5494bbf8f236489c261e5, entries=150, sequenceid=377, filesize=12.0 K 2024-11-23T14:26:08,838 INFO [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for a36a6f46a23b82a88d429516318dcd05 in 1698ms, sequenceid=377, compaction requested=false 2024-11-23T14:26:08,838 DEBUG [StoreCloser-TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/8b9a4536085440c69642aa67c480a522, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/7f02d7fa060445c49a8c1113212bdbc1, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/86ebc0a873dc4a4483a5733acde72e81, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/80762520f40c464495264844b141f777, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/a76a96cbec9b4d838c5f575d3708c9d7, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/dd868ed7718c4340a463cc0c6951ff27, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/8f12c3e5ecbb49d49e15a181521e0a05, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/8cd21b4193b243a1b8d54124c9f87f94, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/c2e10ffca30b4adfb1d07e2fc004a139, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/e87d76f6e9d74cb2bd43dda13aa2ee23, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/f40a29d0dc524157bb22e01bb4739ee6, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/c0a08f3c5ad442eeb911ec04481b9f59, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/147ec2f106714d12983bac7983ff0a5f, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/fb5bc6e9d42e4b44b5ccfa22af70e77e, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/fa4aa82e41ad4687a3253e110be54e31, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/7a53ec4448be4fb787e2e16614c1631b, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/08fc454bb6184943864685a2a45b064b, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/3b705a2fca904b579dc7b56873bc8a3a, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/23e8c6d7e8df48188ca9e7d68665ad05, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/f5bd8bce993f4af89e9533e7db48d9e1, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/9ea6a4a8244e4d3098d41da979be4759, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/1bf4f03a0a8d4fb1a093e0396e9766f1, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/244676194bd945fc998aca2f6c3d301f, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/d02ca8bb91d64dc3ae532a936cb6a83f, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/dc2468669dbf4192b6dfb48156577419] to archive 2024-11-23T14:26:08,839 DEBUG [StoreCloser-TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-23T14:26:08,841 DEBUG [StoreCloser-TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/8b9a4536085440c69642aa67c480a522 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/8b9a4536085440c69642aa67c480a522 2024-11-23T14:26:08,842 DEBUG [StoreCloser-TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/7f02d7fa060445c49a8c1113212bdbc1 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/7f02d7fa060445c49a8c1113212bdbc1 2024-11-23T14:26:08,843 DEBUG [StoreCloser-TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/86ebc0a873dc4a4483a5733acde72e81 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/86ebc0a873dc4a4483a5733acde72e81 2024-11-23T14:26:08,844 DEBUG [StoreCloser-TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/80762520f40c464495264844b141f777 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/80762520f40c464495264844b141f777 2024-11-23T14:26:08,845 DEBUG [StoreCloser-TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/a76a96cbec9b4d838c5f575d3708c9d7 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/a76a96cbec9b4d838c5f575d3708c9d7 2024-11-23T14:26:08,846 DEBUG [StoreCloser-TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/dd868ed7718c4340a463cc0c6951ff27 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/dd868ed7718c4340a463cc0c6951ff27 2024-11-23T14:26:08,847 DEBUG [StoreCloser-TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/8f12c3e5ecbb49d49e15a181521e0a05 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/8f12c3e5ecbb49d49e15a181521e0a05 2024-11-23T14:26:08,848 DEBUG [StoreCloser-TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/8cd21b4193b243a1b8d54124c9f87f94 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/8cd21b4193b243a1b8d54124c9f87f94 2024-11-23T14:26:08,849 DEBUG [StoreCloser-TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/c2e10ffca30b4adfb1d07e2fc004a139 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/c2e10ffca30b4adfb1d07e2fc004a139 2024-11-23T14:26:08,850 DEBUG [StoreCloser-TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/e87d76f6e9d74cb2bd43dda13aa2ee23 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/e87d76f6e9d74cb2bd43dda13aa2ee23 2024-11-23T14:26:08,851 DEBUG [StoreCloser-TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/f40a29d0dc524157bb22e01bb4739ee6 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/f40a29d0dc524157bb22e01bb4739ee6 2024-11-23T14:26:08,852 DEBUG [StoreCloser-TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/c0a08f3c5ad442eeb911ec04481b9f59 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/c0a08f3c5ad442eeb911ec04481b9f59 2024-11-23T14:26:08,853 DEBUG [StoreCloser-TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/147ec2f106714d12983bac7983ff0a5f to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/147ec2f106714d12983bac7983ff0a5f 2024-11-23T14:26:08,853 DEBUG [StoreCloser-TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/fb5bc6e9d42e4b44b5ccfa22af70e77e to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/fb5bc6e9d42e4b44b5ccfa22af70e77e 2024-11-23T14:26:08,854 DEBUG [StoreCloser-TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/fa4aa82e41ad4687a3253e110be54e31 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/fa4aa82e41ad4687a3253e110be54e31 2024-11-23T14:26:08,855 DEBUG [StoreCloser-TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/7a53ec4448be4fb787e2e16614c1631b to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/7a53ec4448be4fb787e2e16614c1631b 2024-11-23T14:26:08,856 DEBUG [StoreCloser-TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/08fc454bb6184943864685a2a45b064b to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/08fc454bb6184943864685a2a45b064b 2024-11-23T14:26:08,857 DEBUG [StoreCloser-TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/3b705a2fca904b579dc7b56873bc8a3a to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/3b705a2fca904b579dc7b56873bc8a3a 2024-11-23T14:26:08,858 DEBUG [StoreCloser-TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/23e8c6d7e8df48188ca9e7d68665ad05 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/23e8c6d7e8df48188ca9e7d68665ad05 2024-11-23T14:26:08,859 DEBUG [StoreCloser-TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/f5bd8bce993f4af89e9533e7db48d9e1 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/f5bd8bce993f4af89e9533e7db48d9e1 2024-11-23T14:26:08,859 DEBUG [StoreCloser-TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/9ea6a4a8244e4d3098d41da979be4759 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/9ea6a4a8244e4d3098d41da979be4759 2024-11-23T14:26:08,860 DEBUG [StoreCloser-TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/1bf4f03a0a8d4fb1a093e0396e9766f1 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/1bf4f03a0a8d4fb1a093e0396e9766f1 2024-11-23T14:26:08,861 DEBUG [StoreCloser-TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/244676194bd945fc998aca2f6c3d301f to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/244676194bd945fc998aca2f6c3d301f 2024-11-23T14:26:08,862 DEBUG [StoreCloser-TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/d02ca8bb91d64dc3ae532a936cb6a83f to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/d02ca8bb91d64dc3ae532a936cb6a83f 2024-11-23T14:26:08,863 DEBUG [StoreCloser-TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/dc2468669dbf4192b6dfb48156577419 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/dc2468669dbf4192b6dfb48156577419 2024-11-23T14:26:08,864 DEBUG [StoreCloser-TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/660b0205af5d41549c89e222c938de2e, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/2ae303836ea34be9afc87a30c9eb7a13, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/f964cc4621f140febad7b00ccd4bc475, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/e080fdca63e84552ad50f80a6ce3d44a, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/b606e7ced1994665a65e30549da2b449, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/500ec3da77a34ecbbba528b6865f204b, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/35ac91d05fab48729159f47a56d86816, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/f12d54b6303c440da62eeaef5075cd73, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/e298e2b41c414ca3a59ae1e621b3255e, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/d49f6b5d2b56432abcfc0f52e42db37a, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/619081c3ebff40e8b5b372900f9db32d, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/6efefc8cc91e439baccd99ec02219605, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/dd3168c6ce1f42538dd1e952fcbc5ac9, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/63c3091562c64c44a32fca07a902fb5c, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/a316e62662304bba97282115d3bd79d0, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/e06d09b41bc64408bdb65b3810afe163, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/746fd8d4727c4bfbb86c15064ee95055, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/c091790ccaca458aa48931d93c5161d4, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/febef176354b465fb8683f9a261d65fe, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/8056a3cffba1405ab1590e1fb330afb0, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/faca79b92da84a35a82d45cccf8c91e4, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/f56b2fee6f024515be8d16bc377d5882, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/c63bba85f2604c93b7e8d63cd17ab4a5, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/703a76eb324f4cfd83e6672815371ce7, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/35433cb7fb6f4c538a5ef5e999dc4ddd] to archive 2024-11-23T14:26:08,865 DEBUG [StoreCloser-TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-23T14:26:08,866 DEBUG [StoreCloser-TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/660b0205af5d41549c89e222c938de2e to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/660b0205af5d41549c89e222c938de2e 2024-11-23T14:26:08,867 DEBUG [StoreCloser-TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/2ae303836ea34be9afc87a30c9eb7a13 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/2ae303836ea34be9afc87a30c9eb7a13 2024-11-23T14:26:08,868 DEBUG [StoreCloser-TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/f964cc4621f140febad7b00ccd4bc475 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/f964cc4621f140febad7b00ccd4bc475 2024-11-23T14:26:08,869 DEBUG [StoreCloser-TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/e080fdca63e84552ad50f80a6ce3d44a to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/e080fdca63e84552ad50f80a6ce3d44a 2024-11-23T14:26:08,870 DEBUG [StoreCloser-TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/b606e7ced1994665a65e30549da2b449 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/b606e7ced1994665a65e30549da2b449 2024-11-23T14:26:08,871 DEBUG [StoreCloser-TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/500ec3da77a34ecbbba528b6865f204b to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/500ec3da77a34ecbbba528b6865f204b 2024-11-23T14:26:08,872 DEBUG [StoreCloser-TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/35ac91d05fab48729159f47a56d86816 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/35ac91d05fab48729159f47a56d86816 2024-11-23T14:26:08,872 DEBUG [StoreCloser-TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/f12d54b6303c440da62eeaef5075cd73 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/f12d54b6303c440da62eeaef5075cd73 2024-11-23T14:26:08,873 DEBUG [StoreCloser-TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/e298e2b41c414ca3a59ae1e621b3255e to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/e298e2b41c414ca3a59ae1e621b3255e 2024-11-23T14:26:08,874 DEBUG [StoreCloser-TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/d49f6b5d2b56432abcfc0f52e42db37a to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/d49f6b5d2b56432abcfc0f52e42db37a 2024-11-23T14:26:08,875 DEBUG [StoreCloser-TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/619081c3ebff40e8b5b372900f9db32d to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/619081c3ebff40e8b5b372900f9db32d 2024-11-23T14:26:08,875 DEBUG [StoreCloser-TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/6efefc8cc91e439baccd99ec02219605 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/6efefc8cc91e439baccd99ec02219605 2024-11-23T14:26:08,876 DEBUG [StoreCloser-TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/dd3168c6ce1f42538dd1e952fcbc5ac9 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/dd3168c6ce1f42538dd1e952fcbc5ac9 2024-11-23T14:26:08,877 DEBUG [StoreCloser-TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/63c3091562c64c44a32fca07a902fb5c to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/63c3091562c64c44a32fca07a902fb5c 2024-11-23T14:26:08,878 DEBUG [StoreCloser-TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/a316e62662304bba97282115d3bd79d0 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/a316e62662304bba97282115d3bd79d0 2024-11-23T14:26:08,879 DEBUG [StoreCloser-TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/e06d09b41bc64408bdb65b3810afe163 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/e06d09b41bc64408bdb65b3810afe163 2024-11-23T14:26:08,880 DEBUG [StoreCloser-TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/746fd8d4727c4bfbb86c15064ee95055 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/746fd8d4727c4bfbb86c15064ee95055 2024-11-23T14:26:08,880 DEBUG [StoreCloser-TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/c091790ccaca458aa48931d93c5161d4 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/c091790ccaca458aa48931d93c5161d4 2024-11-23T14:26:08,881 DEBUG [StoreCloser-TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/febef176354b465fb8683f9a261d65fe to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/febef176354b465fb8683f9a261d65fe 2024-11-23T14:26:08,882 DEBUG [StoreCloser-TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/8056a3cffba1405ab1590e1fb330afb0 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/8056a3cffba1405ab1590e1fb330afb0 2024-11-23T14:26:08,883 DEBUG [StoreCloser-TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/faca79b92da84a35a82d45cccf8c91e4 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/faca79b92da84a35a82d45cccf8c91e4 2024-11-23T14:26:08,884 DEBUG [StoreCloser-TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/f56b2fee6f024515be8d16bc377d5882 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/f56b2fee6f024515be8d16bc377d5882 2024-11-23T14:26:08,885 DEBUG [StoreCloser-TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/c63bba85f2604c93b7e8d63cd17ab4a5 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/c63bba85f2604c93b7e8d63cd17ab4a5 2024-11-23T14:26:08,886 DEBUG [StoreCloser-TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/703a76eb324f4cfd83e6672815371ce7 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/703a76eb324f4cfd83e6672815371ce7 2024-11-23T14:26:08,887 DEBUG [StoreCloser-TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/35433cb7fb6f4c538a5ef5e999dc4ddd to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/35433cb7fb6f4c538a5ef5e999dc4ddd 2024-11-23T14:26:08,888 DEBUG [StoreCloser-TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/5b26b12dff5d470a9f34ed0d875702ea, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/1692f31d1986489f80bfb1803959d95c, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/ed60c40b23b64a94b075891ae055813d, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/78154b6552814181a3a53575262732eb, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/1f49586a29fe48679d909ceeb0805b5d, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/f5070d71c7104d53bedd73258d5d022b, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/0f3695881b19489faba6a663bf85188f, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/0d438be2c0d740f79ac3fe706ccb090e, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/4ff49d8a7f6f4bc98a09c8b88609e86e, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/0c82664b137145c6b121447f5a621d19, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/c0f9708b31ea47eaa535684a6063474e, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/8eb57403117649b8aabb9630efd6fc49, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/3042e046242f4acc83806cb5c3ee4d9d, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/80ecf0bd979847e3944daf3b58a4bcb5, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/e39c0dc2bf67429181e93efad3f33535, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/2cf16f5675ea441286131274963dbf8e, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/18037fa41cb24b1686c7de5bef23c550, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/aa54469a03084c8da913c2a8eadbf2aa, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/27a510ce5bcc4d56b795a29ae08f27a5, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/39508c3260854100a9713cdefcbbd5b9, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/f215aaf8d290437ba61fd5c4e7249c4f, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/6734130028634709a455968f8595878a, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/974ee6d90dab4bcda2d636bc8471cad4, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/8eb224d706284900b84cd38799da16c0, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/f166712c952f4839bbd631f5fc29ef5c] to archive 2024-11-23T14:26:08,889 DEBUG [StoreCloser-TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-23T14:26:08,891 DEBUG [StoreCloser-TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/5b26b12dff5d470a9f34ed0d875702ea to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/5b26b12dff5d470a9f34ed0d875702ea 2024-11-23T14:26:08,891 DEBUG [StoreCloser-TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/1692f31d1986489f80bfb1803959d95c to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/1692f31d1986489f80bfb1803959d95c 2024-11-23T14:26:08,892 DEBUG [StoreCloser-TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/ed60c40b23b64a94b075891ae055813d to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/ed60c40b23b64a94b075891ae055813d 2024-11-23T14:26:08,893 DEBUG [StoreCloser-TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/78154b6552814181a3a53575262732eb to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/78154b6552814181a3a53575262732eb 2024-11-23T14:26:08,894 DEBUG [StoreCloser-TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/1f49586a29fe48679d909ceeb0805b5d to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/1f49586a29fe48679d909ceeb0805b5d 2024-11-23T14:26:08,894 DEBUG [StoreCloser-TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/f5070d71c7104d53bedd73258d5d022b to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/f5070d71c7104d53bedd73258d5d022b 2024-11-23T14:26:08,895 DEBUG [StoreCloser-TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/0f3695881b19489faba6a663bf85188f to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/0f3695881b19489faba6a663bf85188f 2024-11-23T14:26:08,896 DEBUG [StoreCloser-TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/0d438be2c0d740f79ac3fe706ccb090e to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/0d438be2c0d740f79ac3fe706ccb090e 2024-11-23T14:26:08,897 DEBUG [StoreCloser-TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/4ff49d8a7f6f4bc98a09c8b88609e86e to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/4ff49d8a7f6f4bc98a09c8b88609e86e 2024-11-23T14:26:08,898 DEBUG [StoreCloser-TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/0c82664b137145c6b121447f5a621d19 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/0c82664b137145c6b121447f5a621d19 2024-11-23T14:26:08,899 DEBUG [StoreCloser-TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/c0f9708b31ea47eaa535684a6063474e to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/c0f9708b31ea47eaa535684a6063474e 2024-11-23T14:26:08,899 DEBUG [StoreCloser-TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/8eb57403117649b8aabb9630efd6fc49 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/8eb57403117649b8aabb9630efd6fc49 2024-11-23T14:26:08,900 DEBUG [StoreCloser-TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/3042e046242f4acc83806cb5c3ee4d9d to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/3042e046242f4acc83806cb5c3ee4d9d 2024-11-23T14:26:08,901 DEBUG [StoreCloser-TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/80ecf0bd979847e3944daf3b58a4bcb5 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/80ecf0bd979847e3944daf3b58a4bcb5 2024-11-23T14:26:08,902 DEBUG [StoreCloser-TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/e39c0dc2bf67429181e93efad3f33535 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/e39c0dc2bf67429181e93efad3f33535 2024-11-23T14:26:08,903 DEBUG [StoreCloser-TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/2cf16f5675ea441286131274963dbf8e to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/2cf16f5675ea441286131274963dbf8e 2024-11-23T14:26:08,904 DEBUG [StoreCloser-TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/18037fa41cb24b1686c7de5bef23c550 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/18037fa41cb24b1686c7de5bef23c550 2024-11-23T14:26:08,905 DEBUG [StoreCloser-TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/aa54469a03084c8da913c2a8eadbf2aa to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/aa54469a03084c8da913c2a8eadbf2aa 2024-11-23T14:26:08,906 DEBUG [StoreCloser-TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/27a510ce5bcc4d56b795a29ae08f27a5 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/27a510ce5bcc4d56b795a29ae08f27a5 2024-11-23T14:26:08,907 DEBUG [StoreCloser-TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/39508c3260854100a9713cdefcbbd5b9 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/39508c3260854100a9713cdefcbbd5b9 2024-11-23T14:26:08,907 DEBUG [StoreCloser-TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/f215aaf8d290437ba61fd5c4e7249c4f to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/f215aaf8d290437ba61fd5c4e7249c4f 2024-11-23T14:26:08,908 DEBUG [StoreCloser-TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/6734130028634709a455968f8595878a to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/6734130028634709a455968f8595878a 2024-11-23T14:26:08,910 DEBUG [StoreCloser-TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/974ee6d90dab4bcda2d636bc8471cad4 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/974ee6d90dab4bcda2d636bc8471cad4 2024-11-23T14:26:08,911 DEBUG [StoreCloser-TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/8eb224d706284900b84cd38799da16c0 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/8eb224d706284900b84cd38799da16c0 2024-11-23T14:26:08,912 DEBUG [StoreCloser-TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/f166712c952f4839bbd631f5fc29ef5c to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/f166712c952f4839bbd631f5fc29ef5c 2024-11-23T14:26:08,917 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/recovered.edits/380.seqid, newMaxSeqId=380, maxSeqId=4 2024-11-23T14:26:08,917 INFO [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. 2024-11-23T14:26:08,917 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1676): Region close journal for a36a6f46a23b82a88d429516318dcd05: Waiting for close lock at 1732371967139Running coprocessor pre-close hooks at 1732371967139Disabling compacts and flushes for region at 1732371967139Disabling writes for close at 1732371967139Obtaining lock to block concurrent updates at 1732371967140 (+1 ms)Preparing flush snapshotting stores in a36a6f46a23b82a88d429516318dcd05 at 1732371967140Finished memstore snapshotting TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1732371967141 (+1 ms)Flushing stores of TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05. at 1732371967142 (+1 ms)Flushing a36a6f46a23b82a88d429516318dcd05/A: creating writer at 1732371967142Flushing mob file a36a6f46a23b82a88d429516318dcd05/A: appending metadata at 1732371967152 (+10 ms)Flushing mob file a36a6f46a23b82a88d429516318dcd05/A: closing flushed file at 1732371967152Flushing a36a6f46a23b82a88d429516318dcd05/A: appending metadata at 1732371967570 (+418 ms)Flushing a36a6f46a23b82a88d429516318dcd05/A: closing flushed file at 1732371967570Flushing a36a6f46a23b82a88d429516318dcd05/B: creating writer at 1732371967981 (+411 ms)Flushing a36a6f46a23b82a88d429516318dcd05/B: appending metadata at 1732371967985 (+4 ms)Flushing a36a6f46a23b82a88d429516318dcd05/B: closing flushed file at 1732371967985Flushing a36a6f46a23b82a88d429516318dcd05/C: creating writer at 1732371968403 (+418 ms)Flushing a36a6f46a23b82a88d429516318dcd05/C: appending metadata at 1732371968406 (+3 ms)Flushing a36a6f46a23b82a88d429516318dcd05/C: closing flushed file at 1732371968406Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3d3d519f: reopening flushed file at 1732371968820 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7e6c4732: reopening flushed file at 1732371968826 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@61ae2c2a: reopening flushed file at 1732371968831 (+5 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for a36a6f46a23b82a88d429516318dcd05 in 1698ms, sequenceid=377, compaction requested=false at 1732371968838 (+7 ms)Writing region close event to WAL at 1732371968913 (+75 ms)Running coprocessor post-close hooks at 1732371968917 (+4 ms)Closed at 1732371968917 2024-11-23T14:26:08,919 INFO [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] handler.UnassignRegionHandler(157): Closed a36a6f46a23b82a88d429516318dcd05 2024-11-23T14:26:08,919 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=a36a6f46a23b82a88d429516318dcd05, regionState=CLOSED 2024-11-23T14:26:08,920 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=63, ppid=62, state=RUNNABLE, hasLock=false; CloseRegionProcedure a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 because future has completed 2024-11-23T14:26:08,922 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-23T14:26:08,922 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; CloseRegionProcedure a36a6f46a23b82a88d429516318dcd05, server=c6f7b1897123,35385,1732371902767 in 1.9360 sec 2024-11-23T14:26:08,923 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=61 2024-11-23T14:26:08,923 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=61, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a36a6f46a23b82a88d429516318dcd05, UNASSIGN in 1.9400 sec 2024-11-23T14:26:08,925 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-23T14:26:08,925 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9430 sec 2024-11-23T14:26:08,927 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732371968927"}]},"ts":"1732371968927"} 2024-11-23T14:26:08,928 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-23T14:26:08,928 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-23T14:26:08,930 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9680 sec 2024-11-23T14:26:09,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-23T14:26:09,112 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-23T14:26:09,113 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-23T14:26:09,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T14:26:09,118 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=64, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T14:26:09,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-23T14:26:09,120 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=64, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T14:26:09,124 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05 2024-11-23T14:26:09,128 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A, FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B, FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C, FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/recovered.edits] 2024-11-23T14:26:09,133 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/0e37af91b0134712940f875ed8d3237b to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/0e37af91b0134712940f875ed8d3237b 2024-11-23T14:26:09,136 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/229ebd154cad49d493832d4c7355b1ac to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/A/229ebd154cad49d493832d4c7355b1ac 2024-11-23T14:26:09,139 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/12fa08965eb94d5e827d55e30f55646a to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/12fa08965eb94d5e827d55e30f55646a 2024-11-23T14:26:09,141 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/a7d919935b174740a05f91f38ea12021 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/B/a7d919935b174740a05f91f38ea12021 2024-11-23T14:26:09,144 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/3add8bc178b5494bbf8f236489c261e5 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/3add8bc178b5494bbf8f236489c261e5 2024-11-23T14:26:09,146 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/d4061db8a38e4e4b9c89d01c51590ba0 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/C/d4061db8a38e4e4b9c89d01c51590ba0 2024-11-23T14:26:09,150 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/recovered.edits/380.seqid to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05/recovered.edits/380.seqid 2024-11-23T14:26:09,151 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/a36a6f46a23b82a88d429516318dcd05 2024-11-23T14:26:09,151 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-23T14:26:09,151 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-23T14:26:09,153 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-23T14:26:09,158 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112304b9d2dec44e4c4a8fef37b3b557b124_a36a6f46a23b82a88d429516318dcd05 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112304b9d2dec44e4c4a8fef37b3b557b124_a36a6f46a23b82a88d429516318dcd05 2024-11-23T14:26:09,159 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411230a4646e92d6d42d2bc2ce4b06000c2c9_a36a6f46a23b82a88d429516318dcd05 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411230a4646e92d6d42d2bc2ce4b06000c2c9_a36a6f46a23b82a88d429516318dcd05 2024-11-23T14:26:09,161 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123192e55b1f3dd4b7299f2eaef1ee0f59c_a36a6f46a23b82a88d429516318dcd05 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123192e55b1f3dd4b7299f2eaef1ee0f59c_a36a6f46a23b82a88d429516318dcd05 2024-11-23T14:26:09,162 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112319992e64b7e94b2db995c33d49bcbc68_a36a6f46a23b82a88d429516318dcd05 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112319992e64b7e94b2db995c33d49bcbc68_a36a6f46a23b82a88d429516318dcd05 2024-11-23T14:26:09,163 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112320358cbdf6eb450cb14cef83b34cc8aa_a36a6f46a23b82a88d429516318dcd05 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112320358cbdf6eb450cb14cef83b34cc8aa_a36a6f46a23b82a88d429516318dcd05 2024-11-23T14:26:09,165 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123218f4a0ab1ee4164a08f20da7ed863b1_a36a6f46a23b82a88d429516318dcd05 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123218f4a0ab1ee4164a08f20da7ed863b1_a36a6f46a23b82a88d429516318dcd05 2024-11-23T14:26:09,166 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112328a81ccb546448fca1dc869eb0f27370_a36a6f46a23b82a88d429516318dcd05 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112328a81ccb546448fca1dc869eb0f27370_a36a6f46a23b82a88d429516318dcd05 2024-11-23T14:26:09,167 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411232ff5c73239a946ae975768c1e7517ac2_a36a6f46a23b82a88d429516318dcd05 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411232ff5c73239a946ae975768c1e7517ac2_a36a6f46a23b82a88d429516318dcd05 2024-11-23T14:26:09,168 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112349f8bad21ef242d9bf1213f896f3c73f_a36a6f46a23b82a88d429516318dcd05 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112349f8bad21ef242d9bf1213f896f3c73f_a36a6f46a23b82a88d429516318dcd05 2024-11-23T14:26:09,169 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112356e51ef1ad11499a9eb686167d22a5c5_a36a6f46a23b82a88d429516318dcd05 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112356e51ef1ad11499a9eb686167d22a5c5_a36a6f46a23b82a88d429516318dcd05 2024-11-23T14:26:09,170 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112361b9b2f6f9d345fd89c97507d3c3d6e9_a36a6f46a23b82a88d429516318dcd05 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112361b9b2f6f9d345fd89c97507d3c3d6e9_a36a6f46a23b82a88d429516318dcd05 2024-11-23T14:26:09,171 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411236415966190b94bd6913939fb5d32503b_a36a6f46a23b82a88d429516318dcd05 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411236415966190b94bd6913939fb5d32503b_a36a6f46a23b82a88d429516318dcd05 2024-11-23T14:26:09,172 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123768628a7a7164cd1a12c65ea6f937c8e_a36a6f46a23b82a88d429516318dcd05 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123768628a7a7164cd1a12c65ea6f937c8e_a36a6f46a23b82a88d429516318dcd05 2024-11-23T14:26:09,173 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411237b183e6e7b994e2ab82046363b20c54b_a36a6f46a23b82a88d429516318dcd05 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411237b183e6e7b994e2ab82046363b20c54b_a36a6f46a23b82a88d429516318dcd05 2024-11-23T14:26:09,174 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123817b998ee2524b3e88572cbcf4488b45_a36a6f46a23b82a88d429516318dcd05 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123817b998ee2524b3e88572cbcf4488b45_a36a6f46a23b82a88d429516318dcd05 2024-11-23T14:26:09,174 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123a092e3e1248a4953bfae05fcf3a37064_a36a6f46a23b82a88d429516318dcd05 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123a092e3e1248a4953bfae05fcf3a37064_a36a6f46a23b82a88d429516318dcd05 2024-11-23T14:26:09,175 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123da88f18ef85444d2a853b6ca319ce5ae_a36a6f46a23b82a88d429516318dcd05 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123da88f18ef85444d2a853b6ca319ce5ae_a36a6f46a23b82a88d429516318dcd05 2024-11-23T14:26:09,176 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123ddc61febff8c4335a162f0f4d5c3a91f_a36a6f46a23b82a88d429516318dcd05 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123ddc61febff8c4335a162f0f4d5c3a91f_a36a6f46a23b82a88d429516318dcd05 2024-11-23T14:26:09,177 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123e328dbd1f0fc475e9f2377e11eb37a59_a36a6f46a23b82a88d429516318dcd05 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123e328dbd1f0fc475e9f2377e11eb37a59_a36a6f46a23b82a88d429516318dcd05 2024-11-23T14:26:09,178 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123e38066d409ab4b0ead37b6f521960db5_a36a6f46a23b82a88d429516318dcd05 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123e38066d409ab4b0ead37b6f521960db5_a36a6f46a23b82a88d429516318dcd05 2024-11-23T14:26:09,178 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-23T14:26:09,180 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=64, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T14:26:09,182 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-23T14:26:09,185 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-23T14:26:09,186 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=64, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T14:26:09,186 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-23T14:26:09,186 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732371969186"}]},"ts":"9223372036854775807"} 2024-11-23T14:26:09,188 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-23T14:26:09,188 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => a36a6f46a23b82a88d429516318dcd05, NAME => 'TestAcidGuarantees,,1732371934996.a36a6f46a23b82a88d429516318dcd05.', STARTKEY => '', ENDKEY => ''}] 2024-11-23T14:26:09,188 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-23T14:26:09,189 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732371969188"}]},"ts":"9223372036854775807"} 2024-11-23T14:26:09,190 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-23T14:26:09,191 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=64, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T14:26:09,193 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 78 msec 2024-11-23T14:26:09,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-23T14:26:09,231 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-23T14:26:09,231 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-23T14:26:09,239 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=238 (was 239), OpenFileDescriptor=457 (was 457), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=468 (was 507), ProcessCount=11 (was 11), AvailableMemoryMB=3311 (was 3795) 2024-11-23T14:26:09,248 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=238, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=468, ProcessCount=11, AvailableMemoryMB=3308 2024-11-23T14:26:09,249 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-23T14:26:09,250 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-23T14:26:09,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] procedure2.ProcedureExecutor(1139): Stored pid=65, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-23T14:26:09,252 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-23T14:26:09,252 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:26:09,252 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 65 2024-11-23T14:26:09,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-23T14:26:09,253 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-23T14:26:09,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742080_1256 (size=959) 2024-11-23T14:26:09,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-23T14:26:09,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-23T14:26:09,666 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a 2024-11-23T14:26:09,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742081_1257 (size=53) 2024-11-23T14:26:09,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-23T14:26:10,103 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T14:26:10,103 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing baf9478a6ce915eeb361816d4eed6489, disabling compactions & flushes 2024-11-23T14:26:10,103 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:10,103 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:10,103 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. after waiting 0 ms 2024-11-23T14:26:10,103 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:10,103 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:10,104 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for baf9478a6ce915eeb361816d4eed6489: Waiting for close lock at 1732371970103Disabling compacts and flushes for region at 1732371970103Disabling writes for close at 1732371970103Writing region close event to WAL at 1732371970103Closed at 1732371970103 2024-11-23T14:26:10,105 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-23T14:26:10,106 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732371970105"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732371970105"}]},"ts":"1732371970105"} 2024-11-23T14:26:10,108 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-23T14:26:10,109 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-23T14:26:10,109 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732371970109"}]},"ts":"1732371970109"} 2024-11-23T14:26:10,112 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-23T14:26:10,112 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=66, ppid=65, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=baf9478a6ce915eeb361816d4eed6489, ASSIGN}] 2024-11-23T14:26:10,114 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=65, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=baf9478a6ce915eeb361816d4eed6489, ASSIGN 2024-11-23T14:26:10,115 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=66, ppid=65, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=baf9478a6ce915eeb361816d4eed6489, ASSIGN; state=OFFLINE, location=c6f7b1897123,35385,1732371902767; forceNewPlan=false, retain=false 2024-11-23T14:26:10,266 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=66 updating hbase:meta row=baf9478a6ce915eeb361816d4eed6489, regionState=OPENING, regionLocation=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:10,268 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=66, ppid=65, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=baf9478a6ce915eeb361816d4eed6489, ASSIGN because future has completed 2024-11-23T14:26:10,268 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767}] 2024-11-23T14:26:10,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-23T14:26:10,425 INFO [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:10,425 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(7752): Opening region: {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} 2024-11-23T14:26:10,425 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees baf9478a6ce915eeb361816d4eed6489 2024-11-23T14:26:10,425 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T14:26:10,426 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(7794): checking encryption for baf9478a6ce915eeb361816d4eed6489 2024-11-23T14:26:10,426 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(7797): checking classloading for baf9478a6ce915eeb361816d4eed6489 2024-11-23T14:26:10,427 INFO [StoreOpener-baf9478a6ce915eeb361816d4eed6489-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region baf9478a6ce915eeb361816d4eed6489 2024-11-23T14:26:10,429 INFO [StoreOpener-baf9478a6ce915eeb361816d4eed6489-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region baf9478a6ce915eeb361816d4eed6489 columnFamilyName A 2024-11-23T14:26:10,429 DEBUG [StoreOpener-baf9478a6ce915eeb361816d4eed6489-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:26:10,430 INFO [StoreOpener-baf9478a6ce915eeb361816d4eed6489-1 {}] regionserver.HStore(327): Store=baf9478a6ce915eeb361816d4eed6489/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T14:26:10,430 INFO [StoreOpener-baf9478a6ce915eeb361816d4eed6489-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region baf9478a6ce915eeb361816d4eed6489 2024-11-23T14:26:10,431 INFO [StoreOpener-baf9478a6ce915eeb361816d4eed6489-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region baf9478a6ce915eeb361816d4eed6489 columnFamilyName B 2024-11-23T14:26:10,431 DEBUG [StoreOpener-baf9478a6ce915eeb361816d4eed6489-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:26:10,432 INFO [StoreOpener-baf9478a6ce915eeb361816d4eed6489-1 {}] regionserver.HStore(327): Store=baf9478a6ce915eeb361816d4eed6489/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T14:26:10,432 INFO [StoreOpener-baf9478a6ce915eeb361816d4eed6489-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region baf9478a6ce915eeb361816d4eed6489 2024-11-23T14:26:10,433 INFO [StoreOpener-baf9478a6ce915eeb361816d4eed6489-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region baf9478a6ce915eeb361816d4eed6489 columnFamilyName C 2024-11-23T14:26:10,433 DEBUG [StoreOpener-baf9478a6ce915eeb361816d4eed6489-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:26:10,434 INFO [StoreOpener-baf9478a6ce915eeb361816d4eed6489-1 {}] regionserver.HStore(327): Store=baf9478a6ce915eeb361816d4eed6489/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T14:26:10,434 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1038): replaying wal for baf9478a6ce915eeb361816d4eed6489 2024-11-23T14:26:10,435 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489 2024-11-23T14:26:10,435 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489 2024-11-23T14:26:10,436 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1048): stopping wal replay for baf9478a6ce915eeb361816d4eed6489 2024-11-23T14:26:10,436 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1060): Cleaning up temporary data for baf9478a6ce915eeb361816d4eed6489 2024-11-23T14:26:10,437 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-23T14:26:10,438 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1093): writing seq id for baf9478a6ce915eeb361816d4eed6489 2024-11-23T14:26:10,440 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T14:26:10,441 INFO [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1114): Opened baf9478a6ce915eeb361816d4eed6489; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61973967, jitterRate=-0.07651592791080475}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-23T14:26:10,441 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1122): Running coprocessor post-open hooks for baf9478a6ce915eeb361816d4eed6489 2024-11-23T14:26:10,442 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1006): Region open journal for baf9478a6ce915eeb361816d4eed6489: Running coprocessor pre-open hook at 1732371970426Writing region info on filesystem at 1732371970426Initializing all the Stores at 1732371970427 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732371970427Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732371970427Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732371970427Cleaning up temporary data from old regions at 1732371970436 (+9 ms)Running coprocessor post-open hooks at 1732371970441 (+5 ms)Region opened successfully at 1732371970442 (+1 ms) 2024-11-23T14:26:10,443 INFO [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., pid=67, masterSystemTime=1732371970420 2024-11-23T14:26:10,445 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:10,445 INFO [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:10,445 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=66 updating hbase:meta row=baf9478a6ce915eeb361816d4eed6489, regionState=OPEN, openSeqNum=2, regionLocation=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:10,447 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=67, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 because future has completed 2024-11-23T14:26:10,449 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-23T14:26:10,449 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; OpenRegionProcedure baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 in 179 msec 2024-11-23T14:26:10,451 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=66, resume processing ppid=65 2024-11-23T14:26:10,451 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, ppid=65, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=baf9478a6ce915eeb361816d4eed6489, ASSIGN in 337 msec 2024-11-23T14:26:10,452 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-23T14:26:10,452 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732371970452"}]},"ts":"1732371970452"} 2024-11-23T14:26:10,454 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-23T14:26:10,454 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-23T14:26:10,456 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2040 sec 2024-11-23T14:26:11,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-23T14:26:11,400 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-23T14:26:11,401 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b43b976, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:26:11,402 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6f7b1897123,38947,-1 for getting cluster id 2024-11-23T14:26:11,402 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T14:26:11,403 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c88da961-458f-4375-b9d3-275f7f90019e' 2024-11-23T14:26:11,403 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T14:26:11,403 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c88da961-458f-4375-b9d3-275f7f90019e" 2024-11-23T14:26:11,403 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@632b72ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:26:11,403 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6f7b1897123,38947,-1] 2024-11-23T14:26:11,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T14:26:11,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:26:11,405 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51342, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T14:26:11,406 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@618f6660, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:26:11,406 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T14:26:11,407 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6f7b1897123,35385,1732371902767, seqNum=-1] 2024-11-23T14:26:11,408 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T14:26:11,409 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51108, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T14:26:11,410 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-23T14:26:11,411 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is c6f7b1897123,38947,1732371901705 2024-11-23T14:26:11,411 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@738a13d6 2024-11-23T14:26:11,411 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-23T14:26:11,412 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51348, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-23T14:26:11,413 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@429799b1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:26:11,413 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6f7b1897123,38947,-1 for getting cluster id 2024-11-23T14:26:11,413 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T14:26:11,414 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c88da961-458f-4375-b9d3-275f7f90019e' 2024-11-23T14:26:11,414 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T14:26:11,414 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c88da961-458f-4375-b9d3-275f7f90019e" 2024-11-23T14:26:11,415 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@595e4db8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:26:11,415 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6f7b1897123,38947,-1] 2024-11-23T14:26:11,415 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T14:26:11,415 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:26:11,416 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51368, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T14:26:11,416 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fd9773c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:26:11,417 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@480421b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:26:11,417 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6f7b1897123,38947,-1 for getting cluster id 2024-11-23T14:26:11,417 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T14:26:11,418 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c88da961-458f-4375-b9d3-275f7f90019e' 2024-11-23T14:26:11,418 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T14:26:11,418 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c88da961-458f-4375-b9d3-275f7f90019e" 2024-11-23T14:26:11,419 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8219ae3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:26:11,419 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6f7b1897123,38947,-1] 2024-11-23T14:26:11,419 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T14:26:11,419 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:26:11,420 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51386, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T14:26:11,421 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f95f194, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:26:11,435 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e6327b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:26:11,435 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6f7b1897123,38947,-1 for getting cluster id 2024-11-23T14:26:11,435 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T14:26:11,436 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c88da961-458f-4375-b9d3-275f7f90019e' 2024-11-23T14:26:11,436 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T14:26:11,436 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c88da961-458f-4375-b9d3-275f7f90019e" 2024-11-23T14:26:11,436 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b077d9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:26:11,436 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6f7b1897123,38947,-1] 2024-11-23T14:26:11,436 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T14:26:11,437 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:26:11,437 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51412, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T14:26:11,438 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b3dd17d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:26:11,438 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49fdf952, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:26:11,438 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6f7b1897123,38947,-1 for getting cluster id 2024-11-23T14:26:11,438 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T14:26:11,439 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c88da961-458f-4375-b9d3-275f7f90019e' 2024-11-23T14:26:11,439 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T14:26:11,440 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c88da961-458f-4375-b9d3-275f7f90019e" 2024-11-23T14:26:11,440 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@96b940a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:26:11,440 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6f7b1897123,38947,-1] 2024-11-23T14:26:11,440 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T14:26:11,440 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:26:11,441 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51420, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T14:26:11,442 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44bab237, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:26:11,442 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b624df0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:26:11,443 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6f7b1897123,38947,-1 for getting cluster id 2024-11-23T14:26:11,443 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T14:26:11,443 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c88da961-458f-4375-b9d3-275f7f90019e' 2024-11-23T14:26:11,443 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T14:26:11,444 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c88da961-458f-4375-b9d3-275f7f90019e" 2024-11-23T14:26:11,444 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73f9c1d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:26:11,444 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6f7b1897123,38947,-1] 2024-11-23T14:26:11,444 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T14:26:11,444 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:26:11,445 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51440, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T14:26:11,445 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@122fd36e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:26:11,446 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5735c732, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:26:11,446 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6f7b1897123,38947,-1 for getting cluster id 2024-11-23T14:26:11,446 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T14:26:11,446 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c88da961-458f-4375-b9d3-275f7f90019e' 2024-11-23T14:26:11,446 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T14:26:11,447 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c88da961-458f-4375-b9d3-275f7f90019e" 2024-11-23T14:26:11,447 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24f896eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:26:11,447 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6f7b1897123,38947,-1] 2024-11-23T14:26:11,447 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T14:26:11,447 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:26:11,447 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51456, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T14:26:11,448 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1aa943e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:26:11,449 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@212d94ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:26:11,449 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6f7b1897123,38947,-1 for getting cluster id 2024-11-23T14:26:11,449 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T14:26:11,450 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c88da961-458f-4375-b9d3-275f7f90019e' 2024-11-23T14:26:11,450 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T14:26:11,450 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c88da961-458f-4375-b9d3-275f7f90019e" 2024-11-23T14:26:11,450 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f18e94b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:26:11,450 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6f7b1897123,38947,-1] 2024-11-23T14:26:11,450 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T14:26:11,450 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:26:11,451 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51474, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T14:26:11,451 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34574132, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:26:11,452 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3bafd9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:26:11,452 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6f7b1897123,38947,-1 for getting cluster id 2024-11-23T14:26:11,452 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T14:26:11,453 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c88da961-458f-4375-b9d3-275f7f90019e' 2024-11-23T14:26:11,453 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T14:26:11,453 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c88da961-458f-4375-b9d3-275f7f90019e" 2024-11-23T14:26:11,453 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ac1f30b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:26:11,453 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6f7b1897123,38947,-1] 2024-11-23T14:26:11,453 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T14:26:11,454 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:26:11,454 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51486, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T14:26:11,455 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@465c3d70, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:26:11,455 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b15c907, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:26:11,455 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6f7b1897123,38947,-1 for getting cluster id 2024-11-23T14:26:11,455 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T14:26:11,456 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c88da961-458f-4375-b9d3-275f7f90019e' 2024-11-23T14:26:11,456 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T14:26:11,456 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c88da961-458f-4375-b9d3-275f7f90019e" 2024-11-23T14:26:11,456 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cec3d9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:26:11,456 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6f7b1897123,38947,-1] 2024-11-23T14:26:11,457 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T14:26:11,457 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:26:11,457 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51492, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T14:26:11,458 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@114bdb10, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:26:11,459 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12b82d9a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:26:11,459 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6f7b1897123,38947,-1 for getting cluster id 2024-11-23T14:26:11,459 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T14:26:11,459 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c88da961-458f-4375-b9d3-275f7f90019e' 2024-11-23T14:26:11,459 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T14:26:11,460 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c88da961-458f-4375-b9d3-275f7f90019e" 2024-11-23T14:26:11,460 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55064a92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:26:11,460 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6f7b1897123,38947,-1] 2024-11-23T14:26:11,460 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T14:26:11,460 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:26:11,461 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51506, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T14:26:11,461 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@407e26af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:26:11,462 DEBUG [Thread-1189 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T14:26:11,462 DEBUG [Thread-1191 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T14:26:11,463 DEBUG [Thread-1200 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T14:26:11,463 DEBUG [Thread-1204 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T14:26:11,463 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6f7b1897123,35385,1732371902767, seqNum=-1] 2024-11-23T14:26:11,463 DEBUG [Thread-1195 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T14:26:11,463 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T14:26:11,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6f7b1897123,35385,1732371902767, seqNum=-1] 2024-11-23T14:26:11,463 DEBUG [Thread-1208 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T14:26:11,463 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6f7b1897123,35385,1732371902767, seqNum=-1] 2024-11-23T14:26:11,464 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T14:26:11,464 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6f7b1897123,35385,1732371902767, seqNum=-1] 2024-11-23T14:26:11,464 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T14:26:11,464 DEBUG [Thread-1206 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T14:26:11,464 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T14:26:11,465 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51122, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T14:26:11,465 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6f7b1897123,35385,1732371902767, seqNum=-1] 2024-11-23T14:26:11,465 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51138, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T14:26:11,465 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51140, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T14:26:11,465 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6f7b1897123,35385,1732371902767, seqNum=-1] 2024-11-23T14:26:11,465 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T14:26:11,465 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51164, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T14:26:11,465 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T14:26:11,466 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-23T14:26:11,466 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6f7b1897123,35385,1732371902767, seqNum=-1] 2024-11-23T14:26:11,466 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T14:26:11,466 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51170, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T14:26:11,466 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51168, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T14:26:11,467 DEBUG [Thread-1197 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T14:26:11,467 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51182, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T14:26:11,467 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2] 2024-11-23T14:26:11,468 DEBUG [Thread-1193 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T14:26:11,468 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2] 2024-11-23T14:26:11,468 DEBUG [Thread-1202 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T14:26:11,469 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6f7b1897123,35385,1732371902767, seqNum=-1] 2024-11-23T14:26:11,469 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T14:26:11,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-23T14:26:11,469 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2] 2024-11-23T14:26:11,469 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2] 2024-11-23T14:26:11,470 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2] 2024-11-23T14:26:11,470 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2] 2024-11-23T14:26:11,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2] 2024-11-23T14:26:11,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-23T14:26:11,470 DEBUG [Thread-1202 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6f7b1897123,35385,1732371902767, seqNum=-1] 2024-11-23T14:26:11,471 DEBUG [Thread-1202 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T14:26:11,471 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6f7b1897123,35385,1732371902767, seqNum=-1] 2024-11-23T14:26:11,471 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T14:26:11,471 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51188, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T14:26:11,472 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T14:26:11,472 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51196, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T14:26:11,472 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51208, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T14:26:11,473 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T14:26:11,473 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T14:26:11,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2] 2024-11-23T14:26:11,474 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2] 2024-11-23T14:26:11,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2] 2024-11-23T14:26:11,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on baf9478a6ce915eeb361816d4eed6489 2024-11-23T14:26:11,481 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T14:26:11,481 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing baf9478a6ce915eeb361816d4eed6489 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-23T14:26:11,512 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/A/f1fcf5754c4244cf9c2af1f06400b2ef is 50, key is test_row_0/A:col10/1732371971481/Put/seqid=0 2024-11-23T14:26:11,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:11,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51138 deadline: 1732372031510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:11,519 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:11,519 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:11,519 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:11,520 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:11,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51170 deadline: 1732372031513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:11,520 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:11,520 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:11,520 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:11,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:11,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51208 deadline: 1732372031519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:11,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:11,526 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:11,526 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:11,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1732372031519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:11,526 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:11,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:11,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:11,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:11,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:11,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51122 deadline: 1732372031520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:11,527 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:11,527 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:11,527 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:11,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742082_1258 (size=12001) 2024-11-23T14:26:11,544 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/A/f1fcf5754c4244cf9c2af1f06400b2ef 2024-11-23T14:26:11,568 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/B/94006de13e314def8c432f66c67f40ed is 50, key is test_row_0/B:col10/1732371971481/Put/seqid=0 2024-11-23T14:26:11,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-23T14:26:11,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742083_1259 (size=12001) 2024-11-23T14:26:11,627 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-23T14:26:11,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:11,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. as already flushing 2024-11-23T14:26:11,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:11,628 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:11,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:11,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:11,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:11,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51138 deadline: 1732372031630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:11,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:11,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1732372031630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:11,631 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:11,631 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:11,631 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:11,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:11,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:11,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51170 deadline: 1732372031630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:11,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:11,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:11,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:11,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51122 deadline: 1732372031630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:11,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:11,632 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:11,632 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:11,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51208 deadline: 1732372031630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:11,632 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:11,633 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:11,633 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:11,633 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:11,633 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:11,633 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:11,633 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:11,781 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-23T14:26:11,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:11,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. as already flushing 2024-11-23T14:26:11,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:11,782 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:11,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:11,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:11,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-23T14:26:11,842 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:11,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1732372031840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:11,842 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:11,842 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:11,842 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:11,842 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:11,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51170 deadline: 1732372031840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:11,843 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:11,843 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:11,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51138 deadline: 1732372031840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:11,843 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:11,843 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:11,843 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:11,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:11,843 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:11,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51122 deadline: 1732372031840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:11,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51208 deadline: 1732372031841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:11,843 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:11,843 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:11,843 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:11,843 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:11,843 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:11,843 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:11,843 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:11,843 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:11,934 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-23T14:26:11,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:11,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. as already flushing 2024-11-23T14:26:11,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:11,934 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:11,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:11,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:11,993 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/B/94006de13e314def8c432f66c67f40ed 2024-11-23T14:26:12,040 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/C/37dd9dbf45fd494babf5235cce649607 is 50, key is test_row_0/C:col10/1732371971481/Put/seqid=0 2024-11-23T14:26:12,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742084_1260 (size=12001) 2024-11-23T14:26:12,063 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/C/37dd9dbf45fd494babf5235cce649607 2024-11-23T14:26:12,070 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/A/f1fcf5754c4244cf9c2af1f06400b2ef as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/f1fcf5754c4244cf9c2af1f06400b2ef 2024-11-23T14:26:12,077 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/f1fcf5754c4244cf9c2af1f06400b2ef, entries=150, sequenceid=14, filesize=11.7 K 2024-11-23T14:26:12,079 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/B/94006de13e314def8c432f66c67f40ed as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/94006de13e314def8c432f66c67f40ed 2024-11-23T14:26:12,087 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/94006de13e314def8c432f66c67f40ed, entries=150, sequenceid=14, filesize=11.7 K 2024-11-23T14:26:12,087 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-23T14:26:12,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:12,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. as already flushing 2024-11-23T14:26:12,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:12,088 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:12,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:12,089 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/C/37dd9dbf45fd494babf5235cce649607 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/37dd9dbf45fd494babf5235cce649607 2024-11-23T14:26:12,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:12,094 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/37dd9dbf45fd494babf5235cce649607, entries=150, sequenceid=14, filesize=11.7 K 2024-11-23T14:26:12,095 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for baf9478a6ce915eeb361816d4eed6489 in 614ms, sequenceid=14, compaction requested=false 2024-11-23T14:26:12,095 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for baf9478a6ce915eeb361816d4eed6489: 2024-11-23T14:26:12,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-23T14:26:12,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on baf9478a6ce915eeb361816d4eed6489 2024-11-23T14:26:12,152 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T14:26:12,153 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing baf9478a6ce915eeb361816d4eed6489 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-23T14:26:12,157 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/A/4d630290d9d74b9388b6a0497e61488f is 50, key is test_row_0/A:col10/1732371972151/Put/seqid=0 2024-11-23T14:26:12,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:12,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51138 deadline: 1732372032160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:12,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:12,164 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:12,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51170 deadline: 1732372032161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:12,164 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:12,164 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:12,164 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:12,164 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:12,164 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:12,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:12,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1732372032161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:12,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:12,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:12,166 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:12,166 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:12,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51122 deadline: 1732372032162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:12,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:12,166 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:12,166 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:12,166 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:12,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51208 deadline: 1732372032162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:12,166 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:12,166 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:12,166 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:12,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742085_1261 (size=12001) 2024-11-23T14:26:12,170 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/A/4d630290d9d74b9388b6a0497e61488f 2024-11-23T14:26:12,178 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/B/f67eb7a261474af2b142e77e36d3d431 is 50, key is test_row_0/B:col10/1732371972151/Put/seqid=0 2024-11-23T14:26:12,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742086_1262 (size=12001) 2024-11-23T14:26:12,241 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-23T14:26:12,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:12,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. as already flushing 2024-11-23T14:26:12,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:12,241 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:12,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:12,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:12,272 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:12,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51138 deadline: 1732372032270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:12,272 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:12,272 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:12,272 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:12,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51170 deadline: 1732372032270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:12,273 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:12,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:12,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1732372032270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:12,273 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:12,273 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:12,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:12,273 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:12,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51122 deadline: 1732372032270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:12,273 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:12,273 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:12,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:12,273 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:12,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51208 deadline: 1732372032270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:12,273 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:12,273 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:12,273 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:12,273 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:12,274 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:12,274 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:12,394 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-23T14:26:12,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:12,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. as already flushing 2024-11-23T14:26:12,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:12,394 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:12,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:12,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:12,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:12,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51138 deadline: 1732372032480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:12,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:12,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:12,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51170 deadline: 1732372032480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:12,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:12,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:12,482 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:12,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:12,482 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:12,482 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:12,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1732372032480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:12,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:12,482 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:12,482 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:12,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51208 deadline: 1732372032480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:12,482 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:12,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:12,482 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:12,482 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:12,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51122 deadline: 1732372032480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:12,482 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:12,482 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:12,482 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:12,482 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:12,546 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-23T14:26:12,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:12,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. as already flushing 2024-11-23T14:26:12,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:12,547 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:12,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:12,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:12,582 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/B/f67eb7a261474af2b142e77e36d3d431 2024-11-23T14:26:12,589 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/C/0a7b288bb2284b32999cc74acc111b66 is 50, key is test_row_0/C:col10/1732371972151/Put/seqid=0 2024-11-23T14:26:12,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742087_1263 (size=12001) 2024-11-23T14:26:12,593 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/C/0a7b288bb2284b32999cc74acc111b66 2024-11-23T14:26:12,597 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/A/4d630290d9d74b9388b6a0497e61488f as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/4d630290d9d74b9388b6a0497e61488f 2024-11-23T14:26:12,601 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/4d630290d9d74b9388b6a0497e61488f, entries=150, sequenceid=40, filesize=11.7 K 2024-11-23T14:26:12,602 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/B/f67eb7a261474af2b142e77e36d3d431 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/f67eb7a261474af2b142e77e36d3d431 2024-11-23T14:26:12,605 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/f67eb7a261474af2b142e77e36d3d431, entries=150, sequenceid=40, filesize=11.7 K 2024-11-23T14:26:12,606 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/C/0a7b288bb2284b32999cc74acc111b66 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/0a7b288bb2284b32999cc74acc111b66 2024-11-23T14:26:12,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-23T14:26:12,611 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/0a7b288bb2284b32999cc74acc111b66, entries=150, sequenceid=40, filesize=11.7 K 2024-11-23T14:26:12,611 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=53.67 KB/54960 for baf9478a6ce915eeb361816d4eed6489 in 459ms, sequenceid=40, compaction requested=false 2024-11-23T14:26:12,611 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for baf9478a6ce915eeb361816d4eed6489: 2024-11-23T14:26:12,699 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-23T14:26:12,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:12,699 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing baf9478a6ce915eeb361816d4eed6489 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-23T14:26:12,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/A/bb92a85bfaa74c1facd1d6cb6ef76d27 is 50, key is test_row_0/A:col10/1732371972159/Put/seqid=0 2024-11-23T14:26:12,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742088_1264 (size=12001) 2024-11-23T14:26:12,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on baf9478a6ce915eeb361816d4eed6489 2024-11-23T14:26:12,791 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. as already flushing 2024-11-23T14:26:12,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:12,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51122 deadline: 1732372032808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:12,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:12,815 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:12,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1732372032809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:12,815 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:12,815 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:12,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:12,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:12,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:12,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:12,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51138 deadline: 1732372032809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:12,815 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:12,815 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:12,815 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:12,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:12,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51170 deadline: 1732372032814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:12,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:12,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51208 deadline: 1732372032814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:12,819 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:12,819 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:12,820 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:12,820 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:12,820 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:12,820 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:12,922 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:12,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51122 deadline: 1732372032920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:12,922 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:12,922 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:12,922 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:12,922 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:12,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51138 deadline: 1732372032920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:12,922 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:12,923 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:12,922 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:12,923 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:12,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1732372032920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:12,923 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:12,923 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:12,923 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:12,933 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:12,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51170 deadline: 1732372032930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:12,934 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:12,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51208 deadline: 1732372032930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:12,934 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:12,934 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:12,934 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:12,934 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:12,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:12,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:13,108 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/A/bb92a85bfaa74c1facd1d6cb6ef76d27 2024-11-23T14:26:13,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/B/840ebc235b884edc875e75394c7b9336 is 50, key is test_row_0/B:col10/1732371972159/Put/seqid=0 2024-11-23T14:26:13,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:13,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1732372033130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:13,132 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:13,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:13,132 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:13,132 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:13,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51122 deadline: 1732372033130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:13,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:13,133 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:13,133 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:13,133 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:13,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51138 deadline: 1732372033130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:13,133 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:13,133 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:13,133 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:13,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742089_1265 (size=12001) 2024-11-23T14:26:13,143 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:13,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51170 deadline: 1732372033140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:13,143 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:13,143 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:13,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51208 deadline: 1732372033140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:13,143 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:13,143 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:13,143 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:13,143 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:13,143 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:13,447 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:13,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1732372033445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:13,447 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:13,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51138 deadline: 1732372033445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:13,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:13,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:13,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:13,447 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:13,447 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:13,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51122 deadline: 1732372033446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:13,447 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:13,447 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:13,448 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:13,448 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:13,448 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:13,452 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:13,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51170 deadline: 1732372033450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:13,452 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:13,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51208 deadline: 1732372033450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:13,452 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:13,452 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:13,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:13,453 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:13,453 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:13,453 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:13,542 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/B/840ebc235b884edc875e75394c7b9336 2024-11-23T14:26:13,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/C/3382d99ca2d64d7c8945a5a4d7d6adbf is 50, key is test_row_0/C:col10/1732371972159/Put/seqid=0 2024-11-23T14:26:13,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742090_1266 (size=12001) 2024-11-23T14:26:13,557 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/C/3382d99ca2d64d7c8945a5a4d7d6adbf 2024-11-23T14:26:13,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/A/bb92a85bfaa74c1facd1d6cb6ef76d27 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/bb92a85bfaa74c1facd1d6cb6ef76d27 2024-11-23T14:26:13,575 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/bb92a85bfaa74c1facd1d6cb6ef76d27, entries=150, sequenceid=51, filesize=11.7 K 2024-11-23T14:26:13,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/B/840ebc235b884edc875e75394c7b9336 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/840ebc235b884edc875e75394c7b9336 2024-11-23T14:26:13,584 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/840ebc235b884edc875e75394c7b9336, entries=150, sequenceid=51, filesize=11.7 K 2024-11-23T14:26:13,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/C/3382d99ca2d64d7c8945a5a4d7d6adbf as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/3382d99ca2d64d7c8945a5a4d7d6adbf 2024-11-23T14:26:13,592 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/3382d99ca2d64d7c8945a5a4d7d6adbf, entries=150, sequenceid=51, filesize=11.7 K 2024-11-23T14:26:13,594 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for baf9478a6ce915eeb361816d4eed6489 in 894ms, sequenceid=51, compaction requested=true 2024-11-23T14:26:13,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for baf9478a6ce915eeb361816d4eed6489: 2024-11-23T14:26:13,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:13,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-23T14:26:13,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-23T14:26:13,598 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-23T14:26:13,598 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1220 sec 2024-11-23T14:26:13,601 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 2.1330 sec 2024-11-23T14:26:13,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-23T14:26:13,620 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T14:26:13,621 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-23T14:26:13,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-23T14:26:13,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-23T14:26:13,622 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T14:26:13,623 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T14:26:13,623 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T14:26:13,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-23T14:26:13,775 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-23T14:26:13,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:13,775 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing baf9478a6ce915eeb361816d4eed6489 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-23T14:26:13,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/A/342a3cd26c824a659c805c081b9aa1ef is 50, key is test_row_0/A:col10/1732371972813/Put/seqid=0 2024-11-23T14:26:13,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742091_1267 (size=12001) 2024-11-23T14:26:13,784 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/A/342a3cd26c824a659c805c081b9aa1ef 2024-11-23T14:26:13,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/B/796c4c600e71424f92aa360b64276fe3 is 50, key is test_row_0/B:col10/1732371972813/Put/seqid=0 2024-11-23T14:26:13,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742092_1268 (size=12001) 2024-11-23T14:26:13,798 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/B/796c4c600e71424f92aa360b64276fe3 2024-11-23T14:26:13,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/C/367417322d7747e283ddfbd4c74d2e3d is 50, key is test_row_0/C:col10/1732371972813/Put/seqid=0 2024-11-23T14:26:13,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742093_1269 (size=12001) 2024-11-23T14:26:13,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-23T14:26:13,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on baf9478a6ce915eeb361816d4eed6489 2024-11-23T14:26:13,961 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. as already flushing 2024-11-23T14:26:13,969 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:13,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51170 deadline: 1732372033963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:13,969 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:13,969 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:13,969 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:13,969 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:13,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51208 deadline: 1732372033964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:13,970 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:13,970 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:13,970 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:13,970 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:13,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1732372033967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:13,971 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:13,971 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:13,971 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:13,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:13,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51138 deadline: 1732372033968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:13,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:13,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51122 deadline: 1732372033969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:13,973 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:13,973 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:13,973 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:13,974 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:13,974 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:13,974 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:14,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:14,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51170 deadline: 1732372034080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:14,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:14,083 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:14,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51138 deadline: 1732372034081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:14,083 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:14,084 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:14,084 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:14,084 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:14,084 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:14,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51122 deadline: 1732372034081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:14,084 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:14,084 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:14,084 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:14,084 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:14,084 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:14,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1732372034081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:14,084 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:14,084 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:14,084 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:14,084 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:14,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51208 deadline: 1732372034081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:14,085 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:14,085 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:14,085 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:14,209 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/C/367417322d7747e283ddfbd4c74d2e3d 2024-11-23T14:26:14,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/A/342a3cd26c824a659c805c081b9aa1ef as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/342a3cd26c824a659c805c081b9aa1ef 2024-11-23T14:26:14,226 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/342a3cd26c824a659c805c081b9aa1ef, entries=150, sequenceid=76, filesize=11.7 K 2024-11-23T14:26:14,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/B/796c4c600e71424f92aa360b64276fe3 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/796c4c600e71424f92aa360b64276fe3 2024-11-23T14:26:14,234 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/796c4c600e71424f92aa360b64276fe3, entries=150, sequenceid=76, filesize=11.7 K 2024-11-23T14:26:14,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/C/367417322d7747e283ddfbd4c74d2e3d as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/367417322d7747e283ddfbd4c74d2e3d 2024-11-23T14:26:14,241 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/367417322d7747e283ddfbd4c74d2e3d, entries=150, sequenceid=76, filesize=11.7 K 2024-11-23T14:26:14,242 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for baf9478a6ce915eeb361816d4eed6489 in 467ms, sequenceid=76, compaction requested=true 2024-11-23T14:26:14,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for baf9478a6ce915eeb361816d4eed6489: 2024-11-23T14:26:14,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:14,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-23T14:26:14,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-23T14:26:14,245 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-23T14:26:14,246 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 621 msec 2024-11-23T14:26:14,248 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 625 msec 2024-11-23T14:26:14,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-23T14:26:14,250 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T14:26:14,250 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-23T14:26:14,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-23T14:26:14,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-23T14:26:14,254 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T14:26:14,255 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T14:26:14,255 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T14:26:14,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on baf9478a6ce915eeb361816d4eed6489 2024-11-23T14:26:14,293 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T14:26:14,293 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing baf9478a6ce915eeb361816d4eed6489 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-23T14:26:14,314 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/A/4a8599eac41144e7807ba23235bde840 is 50, key is test_row_0/A:col10/1732371973966/Put/seqid=0 2024-11-23T14:26:14,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:14,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51122 deadline: 1732372034319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:14,328 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:14,328 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:14,328 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:14,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:14,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1732372034321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:14,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:14,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:14,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:14,330 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:14,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51208 deadline: 1732372034323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:14,331 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:14,331 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:14,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:14,331 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:14,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51170 deadline: 1732372034324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:14,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:14,331 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:14,331 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:14,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51138 deadline: 1732372034325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:14,331 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:14,332 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:14,332 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:14,332 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:14,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742094_1270 (size=12001) 2024-11-23T14:26:14,342 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/A/4a8599eac41144e7807ba23235bde840 2024-11-23T14:26:14,349 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/B/74c19b0b15534370a632263b234a6632 is 50, key is test_row_0/B:col10/1732371973966/Put/seqid=0 2024-11-23T14:26:14,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742095_1271 (size=12001) 2024-11-23T14:26:14,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-23T14:26:14,410 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-23T14:26:14,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:14,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. as already flushing 2024-11-23T14:26:14,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:14,411 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:14,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:14,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:14,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:14,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1732372034430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:14,435 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:14,435 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:14,435 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:14,438 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-23T14:26:14,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:14,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51122 deadline: 1732372034440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:14,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:14,444 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:14,444 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:14,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51208 deadline: 1732372034440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:14,444 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:14,444 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:14,444 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:14,444 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:14,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:14,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51170 deadline: 1732372034440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:14,446 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:14,446 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:14,446 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:14,447 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:14,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51138 deadline: 1732372034441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:14,447 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:14,447 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:14,447 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:14,565 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-23T14:26:14,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:14,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. as already flushing 2024-11-23T14:26:14,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:14,565 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:14,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:14,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:14,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-23T14:26:14,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:14,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1732372034640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:14,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:14,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:14,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:14,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:14,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51208 deadline: 1732372034650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:14,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:14,654 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:14,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51170 deadline: 1732372034650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:14,654 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:14,654 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:14,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:14,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:14,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:14,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:14,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51122 deadline: 1732372034654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:14,657 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:14,657 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:14,657 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:14,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:14,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51138 deadline: 1732372034660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:14,663 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:14,663 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:14,663 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:14,718 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-23T14:26:14,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:14,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. as already flushing 2024-11-23T14:26:14,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:14,719 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:14,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:14,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:14,758 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/B/74c19b0b15534370a632263b234a6632 2024-11-23T14:26:14,768 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/C/be29c0ba59e24426b15bcb21e3385540 is 50, key is test_row_0/C:col10/1732371973966/Put/seqid=0 2024-11-23T14:26:14,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742096_1272 (size=12001) 2024-11-23T14:26:14,777 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/C/be29c0ba59e24426b15bcb21e3385540 2024-11-23T14:26:14,782 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/A/4a8599eac41144e7807ba23235bde840 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/4a8599eac41144e7807ba23235bde840 2024-11-23T14:26:14,793 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/4a8599eac41144e7807ba23235bde840, entries=150, sequenceid=89, filesize=11.7 K 2024-11-23T14:26:14,795 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/B/74c19b0b15534370a632263b234a6632 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/74c19b0b15534370a632263b234a6632 2024-11-23T14:26:14,810 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/74c19b0b15534370a632263b234a6632, entries=150, sequenceid=89, filesize=11.7 K 2024-11-23T14:26:14,811 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/C/be29c0ba59e24426b15bcb21e3385540 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/be29c0ba59e24426b15bcb21e3385540 2024-11-23T14:26:14,835 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/be29c0ba59e24426b15bcb21e3385540, entries=150, sequenceid=89, filesize=11.7 K 2024-11-23T14:26:14,836 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for baf9478a6ce915eeb361816d4eed6489 in 543ms, sequenceid=89, compaction requested=true 2024-11-23T14:26:14,837 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for baf9478a6ce915eeb361816d4eed6489: 2024-11-23T14:26:14,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store baf9478a6ce915eeb361816d4eed6489:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T14:26:14,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:26:14,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store baf9478a6ce915eeb361816d4eed6489:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T14:26:14,837 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-23T14:26:14,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:26:14,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store baf9478a6ce915eeb361816d4eed6489:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T14:26:14,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-23T14:26:14,837 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-23T14:26:14,839 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60005 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-23T14:26:14,839 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1541): baf9478a6ce915eeb361816d4eed6489/A is initiating minor compaction (all files) 2024-11-23T14:26:14,839 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of baf9478a6ce915eeb361816d4eed6489/A in TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:14,839 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/f1fcf5754c4244cf9c2af1f06400b2ef, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/4d630290d9d74b9388b6a0497e61488f, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/bb92a85bfaa74c1facd1d6cb6ef76d27, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/342a3cd26c824a659c805c081b9aa1ef, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/4a8599eac41144e7807ba23235bde840] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp, totalSize=58.6 K 2024-11-23T14:26:14,840 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60005 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-23T14:26:14,840 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1541): baf9478a6ce915eeb361816d4eed6489/B is initiating minor compaction (all files) 2024-11-23T14:26:14,840 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of baf9478a6ce915eeb361816d4eed6489/B in TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:14,840 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/94006de13e314def8c432f66c67f40ed, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/f67eb7a261474af2b142e77e36d3d431, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/840ebc235b884edc875e75394c7b9336, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/796c4c600e71424f92aa360b64276fe3, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/74c19b0b15534370a632263b234a6632] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp, totalSize=58.6 K 2024-11-23T14:26:14,840 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting f1fcf5754c4244cf9c2af1f06400b2ef, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732371971475 2024-11-23T14:26:14,841 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 94006de13e314def8c432f66c67f40ed, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732371971475 2024-11-23T14:26:14,843 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d630290d9d74b9388b6a0497e61488f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732371971516 2024-11-23T14:26:14,844 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting f67eb7a261474af2b142e77e36d3d431, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732371971516 2024-11-23T14:26:14,844 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting bb92a85bfaa74c1facd1d6cb6ef76d27, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732371972153 2024-11-23T14:26:14,844 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 840ebc235b884edc875e75394c7b9336, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732371972153 2024-11-23T14:26:14,844 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 342a3cd26c824a659c805c081b9aa1ef, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1732371972807 2024-11-23T14:26:14,844 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 796c4c600e71424f92aa360b64276fe3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1732371972807 2024-11-23T14:26:14,845 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a8599eac41144e7807ba23235bde840, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1732371973962 2024-11-23T14:26:14,845 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 74c19b0b15534370a632263b234a6632, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1732371973962 2024-11-23T14:26:14,862 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): baf9478a6ce915eeb361816d4eed6489#B#compaction#228 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:26:14,862 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/B/dd5847abb25f4b5589ebd148d3d547ea is 50, key is test_row_0/B:col10/1732371973966/Put/seqid=0 2024-11-23T14:26:14,871 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-23T14:26:14,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:14,872 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing baf9478a6ce915eeb361816d4eed6489 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-23T14:26:14,876 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): baf9478a6ce915eeb361816d4eed6489#A#compaction#229 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:26:14,877 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/A/dadacaeec57a4a73b634018f1bc5d500 is 50, key is test_row_0/A:col10/1732371973966/Put/seqid=0 2024-11-23T14:26:14,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/A/397999acefda4662a1fc269d06d2369f is 50, key is test_row_0/A:col10/1732371974317/Put/seqid=0 2024-11-23T14:26:14,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-23T14:26:14,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742097_1273 (size=12173) 2024-11-23T14:26:14,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742098_1274 (size=12173) 2024-11-23T14:26:14,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742099_1275 (size=12001) 2024-11-23T14:26:14,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on baf9478a6ce915eeb361816d4eed6489 2024-11-23T14:26:14,954 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. as already flushing 2024-11-23T14:26:14,975 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:14,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51138 deadline: 1732372034970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:14,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:14,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51170 deadline: 1732372034970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:14,975 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:14,975 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:14,975 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:14,975 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:14,975 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:14,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:14,975 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:14,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51208 deadline: 1732372034971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:14,975 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:14,976 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:14,976 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:14,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:14,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1732372034974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:14,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:14,979 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:14,979 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:14,979 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:14,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51122 deadline: 1732372034975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:14,979 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:14,979 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:14,980 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:15,084 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:15,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51170 deadline: 1732372035080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:15,084 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:15,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51208 deadline: 1732372035080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:15,084 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:15,084 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:15,084 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:15,085 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:15,085 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:15,085 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:15,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:15,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1732372035090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:15,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:15,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:15,095 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:15,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:15,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51122 deadline: 1732372035090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:15,095 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:15,095 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:15,095 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:15,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:15,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51170 deadline: 1732372035290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:15,292 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:15,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51208 deadline: 1732372035290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:15,292 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:15,292 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:15,292 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:15,292 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:15,292 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:15,292 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:15,299 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/B/dd5847abb25f4b5589ebd148d3d547ea as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/dd5847abb25f4b5589ebd148d3d547ea 2024-11-23T14:26:15,302 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:15,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51122 deadline: 1732372035300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:15,303 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:15,303 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:15,303 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:15,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1732372035300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:15,303 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:15,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:15,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:15,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:15,304 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in baf9478a6ce915eeb361816d4eed6489/B of baf9478a6ce915eeb361816d4eed6489 into dd5847abb25f4b5589ebd148d3d547ea(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:26:15,304 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for baf9478a6ce915eeb361816d4eed6489: 2024-11-23T14:26:15,304 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., storeName=baf9478a6ce915eeb361816d4eed6489/B, priority=11, startTime=1732371974837; duration=0sec 2024-11-23T14:26:15,304 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:26:15,304 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: baf9478a6ce915eeb361816d4eed6489:B 2024-11-23T14:26:15,304 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-23T14:26:15,306 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60005 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-23T14:26:15,306 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1541): baf9478a6ce915eeb361816d4eed6489/C is initiating minor compaction (all files) 2024-11-23T14:26:15,306 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of baf9478a6ce915eeb361816d4eed6489/C in TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:15,306 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/37dd9dbf45fd494babf5235cce649607, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/0a7b288bb2284b32999cc74acc111b66, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/3382d99ca2d64d7c8945a5a4d7d6adbf, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/367417322d7747e283ddfbd4c74d2e3d, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/be29c0ba59e24426b15bcb21e3385540] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp, totalSize=58.6 K 2024-11-23T14:26:15,307 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 37dd9dbf45fd494babf5235cce649607, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732371971475 2024-11-23T14:26:15,307 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a7b288bb2284b32999cc74acc111b66, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732371971516 2024-11-23T14:26:15,307 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3382d99ca2d64d7c8945a5a4d7d6adbf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732371972153 2024-11-23T14:26:15,307 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 367417322d7747e283ddfbd4c74d2e3d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1732371972807 2024-11-23T14:26:15,308 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting be29c0ba59e24426b15bcb21e3385540, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1732371973962 2024-11-23T14:26:15,308 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/A/dadacaeec57a4a73b634018f1bc5d500 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/dadacaeec57a4a73b634018f1bc5d500 2024-11-23T14:26:15,310 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/A/397999acefda4662a1fc269d06d2369f 2024-11-23T14:26:15,313 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in baf9478a6ce915eeb361816d4eed6489/A of baf9478a6ce915eeb361816d4eed6489 into dadacaeec57a4a73b634018f1bc5d500(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:26:15,313 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for baf9478a6ce915eeb361816d4eed6489: 2024-11-23T14:26:15,313 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., storeName=baf9478a6ce915eeb361816d4eed6489/A, priority=11, startTime=1732371974837; duration=0sec 2024-11-23T14:26:15,313 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:26:15,313 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: baf9478a6ce915eeb361816d4eed6489:A 2024-11-23T14:26:15,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/B/07bf96bc69e94099a15b52abf2258176 is 50, key is test_row_0/B:col10/1732371974317/Put/seqid=0 2024-11-23T14:26:15,318 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): baf9478a6ce915eeb361816d4eed6489#C#compaction#232 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:26:15,318 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/C/7ca348205a0b4c2cb4173c49f2ce76f4 is 50, key is test_row_0/C:col10/1732371973966/Put/seqid=0 2024-11-23T14:26:15,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742100_1276 (size=12001) 2024-11-23T14:26:15,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742101_1277 (size=12173) 2024-11-23T14:26:15,329 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/C/7ca348205a0b4c2cb4173c49f2ce76f4 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/7ca348205a0b4c2cb4173c49f2ce76f4 2024-11-23T14:26:15,333 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in baf9478a6ce915eeb361816d4eed6489/C of baf9478a6ce915eeb361816d4eed6489 into 7ca348205a0b4c2cb4173c49f2ce76f4(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:26:15,333 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for baf9478a6ce915eeb361816d4eed6489: 2024-11-23T14:26:15,333 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., storeName=baf9478a6ce915eeb361816d4eed6489/C, priority=11, startTime=1732371974837; duration=0sec 2024-11-23T14:26:15,333 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:26:15,333 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: baf9478a6ce915eeb361816d4eed6489:C 2024-11-23T14:26:15,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-23T14:26:15,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:15,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51138 deadline: 1732372035480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:15,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:15,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:15,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:15,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:15,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51170 deadline: 1732372035600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:15,602 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:15,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:15,602 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:15,602 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:15,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51208 deadline: 1732372035600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:15,602 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:15,602 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:15,602 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:15,612 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:15,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51122 deadline: 1732372035610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:15,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:15,613 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:15,613 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:15,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1732372035610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:15,613 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:15,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:15,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:15,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:15,724 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/B/07bf96bc69e94099a15b52abf2258176 2024-11-23T14:26:15,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/C/b192a9ef10554cc488020be2f9359663 is 50, key is test_row_0/C:col10/1732371974317/Put/seqid=0 2024-11-23T14:26:15,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742102_1278 (size=12001) 2024-11-23T14:26:16,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:16,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51208 deadline: 1732372036110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:16,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:16,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51170 deadline: 1732372036110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:16,113 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:16,113 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:16,113 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:16,113 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:16,113 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:16,113 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:16,121 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:16,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51122 deadline: 1732372036120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:16,122 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:16,122 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:16,122 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:16,122 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:16,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1732372036120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:16,122 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:16,122 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:16,122 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:16,133 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/C/b192a9ef10554cc488020be2f9359663 2024-11-23T14:26:16,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/A/397999acefda4662a1fc269d06d2369f as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/397999acefda4662a1fc269d06d2369f 2024-11-23T14:26:16,142 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/397999acefda4662a1fc269d06d2369f, entries=150, sequenceid=112, filesize=11.7 K 2024-11-23T14:26:16,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/B/07bf96bc69e94099a15b52abf2258176 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/07bf96bc69e94099a15b52abf2258176 2024-11-23T14:26:16,146 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/07bf96bc69e94099a15b52abf2258176, entries=150, sequenceid=112, filesize=11.7 K 2024-11-23T14:26:16,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/C/b192a9ef10554cc488020be2f9359663 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/b192a9ef10554cc488020be2f9359663 2024-11-23T14:26:16,151 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/b192a9ef10554cc488020be2f9359663, entries=150, sequenceid=112, filesize=11.7 K 2024-11-23T14:26:16,152 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for baf9478a6ce915eeb361816d4eed6489 in 1281ms, sequenceid=112, compaction requested=false 2024-11-23T14:26:16,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for baf9478a6ce915eeb361816d4eed6489: 2024-11-23T14:26:16,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:16,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-23T14:26:16,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-23T14:26:16,155 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-23T14:26:16,155 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8990 sec 2024-11-23T14:26:16,157 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.9050 sec 2024-11-23T14:26:16,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-23T14:26:16,400 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T14:26:16,400 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-23T14:26:16,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-23T14:26:16,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-23T14:26:16,402 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T14:26:16,403 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T14:26:16,403 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T14:26:16,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on baf9478a6ce915eeb361816d4eed6489 2024-11-23T14:26:16,502 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T14:26:16,502 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing baf9478a6ce915eeb361816d4eed6489 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-23T14:26:16,506 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/A/ce872b59e4294deaa509506cea8563b4 is 50, key is test_row_0/A:col10/1732371976501/Put/seqid=0 2024-11-23T14:26:16,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742103_1279 (size=12051) 2024-11-23T14:26:16,509 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/A/ce872b59e4294deaa509506cea8563b4 2024-11-23T14:26:16,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-23T14:26:16,516 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/B/cb96316e344c443497f62ea7439a84e2 is 50, key is test_row_0/B:col10/1732371976501/Put/seqid=0 2024-11-23T14:26:16,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742104_1280 (size=12051) 2024-11-23T14:26:16,555 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-23T14:26:16,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:16,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. as already flushing 2024-11-23T14:26:16,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:16,556 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:16,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:16,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:16,593 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:16,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51138 deadline: 1732372036590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:16,593 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:16,593 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:16,593 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:16,702 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:16,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51138 deadline: 1732372036700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:16,702 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:16,702 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:16,702 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:16,708 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-23T14:26:16,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:16,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. as already flushing 2024-11-23T14:26:16,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:16,709 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:16,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:16,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:16,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-23T14:26:16,861 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-23T14:26:16,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:16,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. as already flushing 2024-11-23T14:26:16,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:16,862 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:16,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:16,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:16,912 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:16,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51138 deadline: 1732372036910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:16,912 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:16,912 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:16,912 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:16,922 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/B/cb96316e344c443497f62ea7439a84e2 2024-11-23T14:26:16,933 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/C/51081ca63d9640a6818dd51fc1feab93 is 50, key is test_row_0/C:col10/1732371976501/Put/seqid=0 2024-11-23T14:26:16,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742105_1281 (size=12051) 2024-11-23T14:26:17,015 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-23T14:26:17,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:17,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. as already flushing 2024-11-23T14:26:17,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:17,015 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:17,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:17,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:17,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-23T14:26:17,123 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:17,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51208 deadline: 1732372037120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:17,123 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to c6f7b1897123,35385,1732371902767 for 'test_row_1' in baf9478a6ce915eeb361816d4eed6489 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2152 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-23T14:26:17,124 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:17,124 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:17,124 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:17,133 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:17,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51170 deadline: 1732372037130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:17,133 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:17,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51122 deadline: 1732372037130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:17,133 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to c6f7b1897123,35385,1732371902767 for 'test_row_1' in baf9478a6ce915eeb361816d4eed6489 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-23T14:26:17,134 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:17,134 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:17,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1732372037130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:17,134 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:17,134 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:17,134 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to c6f7b1897123,35385,1732371902767 for 'test_row_1' in baf9478a6ce915eeb361816d4eed6489 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-23T14:26:17,134 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:17,134 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:17,134 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:17,134 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to c6f7b1897123,35385,1732371902767 for 'test_row_0' in baf9478a6ce915eeb361816d4eed6489 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-23T14:26:17,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:17,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:17,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:17,168 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-23T14:26:17,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:17,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. as already flushing 2024-11-23T14:26:17,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:17,168 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:17,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:17,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:17,223 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:17,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51138 deadline: 1732372037220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:17,223 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:17,223 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:17,223 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:17,322 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-23T14:26:17,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:17,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. as already flushing 2024-11-23T14:26:17,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:17,323 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:17,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:17,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:17,341 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/C/51081ca63d9640a6818dd51fc1feab93 2024-11-23T14:26:17,346 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/A/ce872b59e4294deaa509506cea8563b4 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/ce872b59e4294deaa509506cea8563b4 2024-11-23T14:26:17,352 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/ce872b59e4294deaa509506cea8563b4, entries=150, sequenceid=129, filesize=11.8 K 2024-11-23T14:26:17,353 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/B/cb96316e344c443497f62ea7439a84e2 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/cb96316e344c443497f62ea7439a84e2 2024-11-23T14:26:17,358 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/cb96316e344c443497f62ea7439a84e2, entries=150, sequenceid=129, filesize=11.8 K 2024-11-23T14:26:17,359 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/C/51081ca63d9640a6818dd51fc1feab93 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/51081ca63d9640a6818dd51fc1feab93 2024-11-23T14:26:17,366 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/51081ca63d9640a6818dd51fc1feab93, entries=150, sequenceid=129, filesize=11.8 K 2024-11-23T14:26:17,367 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for baf9478a6ce915eeb361816d4eed6489 in 865ms, sequenceid=129, compaction requested=true 2024-11-23T14:26:17,367 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for baf9478a6ce915eeb361816d4eed6489: 2024-11-23T14:26:17,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store baf9478a6ce915eeb361816d4eed6489:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T14:26:17,367 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:26:17,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:26:17,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store baf9478a6ce915eeb361816d4eed6489:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T14:26:17,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:26:17,367 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:26:17,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store baf9478a6ce915eeb361816d4eed6489:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T14:26:17,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:26:17,368 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36225 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:26:17,368 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36225 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:26:17,368 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1541): baf9478a6ce915eeb361816d4eed6489/B is initiating minor compaction (all files) 2024-11-23T14:26:17,368 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1541): baf9478a6ce915eeb361816d4eed6489/A is initiating minor compaction (all files) 2024-11-23T14:26:17,368 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of baf9478a6ce915eeb361816d4eed6489/A in TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:17,368 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of baf9478a6ce915eeb361816d4eed6489/B in TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:17,368 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/dadacaeec57a4a73b634018f1bc5d500, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/397999acefda4662a1fc269d06d2369f, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/ce872b59e4294deaa509506cea8563b4] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp, totalSize=35.4 K 2024-11-23T14:26:17,368 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/dd5847abb25f4b5589ebd148d3d547ea, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/07bf96bc69e94099a15b52abf2258176, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/cb96316e344c443497f62ea7439a84e2] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp, totalSize=35.4 K 2024-11-23T14:26:17,369 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting dadacaeec57a4a73b634018f1bc5d500, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1732371973962 2024-11-23T14:26:17,369 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting dd5847abb25f4b5589ebd148d3d547ea, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1732371973962 2024-11-23T14:26:17,369 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 397999acefda4662a1fc269d06d2369f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1732371974317 2024-11-23T14:26:17,369 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 07bf96bc69e94099a15b52abf2258176, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1732371974317 2024-11-23T14:26:17,369 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce872b59e4294deaa509506cea8563b4, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1732371974957 2024-11-23T14:26:17,369 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting cb96316e344c443497f62ea7439a84e2, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1732371974957 2024-11-23T14:26:17,393 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): baf9478a6ce915eeb361816d4eed6489#B#compaction#237 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:26:17,394 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/B/39736117dee5489cbe4147cad9cf31ea is 50, key is test_row_0/B:col10/1732371976501/Put/seqid=0 2024-11-23T14:26:17,395 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): baf9478a6ce915eeb361816d4eed6489#A#compaction#238 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:26:17,395 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/A/e700eca174de47799af9363037b2cb4c is 50, key is test_row_0/A:col10/1732371976501/Put/seqid=0 2024-11-23T14:26:17,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742106_1282 (size=12325) 2024-11-23T14:26:17,421 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/B/39736117dee5489cbe4147cad9cf31ea as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/39736117dee5489cbe4147cad9cf31ea 2024-11-23T14:26:17,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742107_1283 (size=12325) 2024-11-23T14:26:17,427 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in baf9478a6ce915eeb361816d4eed6489/B of baf9478a6ce915eeb361816d4eed6489 into 39736117dee5489cbe4147cad9cf31ea(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:26:17,427 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for baf9478a6ce915eeb361816d4eed6489: 2024-11-23T14:26:17,427 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., storeName=baf9478a6ce915eeb361816d4eed6489/B, priority=13, startTime=1732371977367; duration=0sec 2024-11-23T14:26:17,427 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:26:17,427 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: baf9478a6ce915eeb361816d4eed6489:B 2024-11-23T14:26:17,427 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:26:17,428 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36225 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:26:17,428 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1541): baf9478a6ce915eeb361816d4eed6489/C is initiating minor compaction (all files) 2024-11-23T14:26:17,428 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of baf9478a6ce915eeb361816d4eed6489/C in TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:17,429 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/7ca348205a0b4c2cb4173c49f2ce76f4, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/b192a9ef10554cc488020be2f9359663, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/51081ca63d9640a6818dd51fc1feab93] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp, totalSize=35.4 K 2024-11-23T14:26:17,429 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ca348205a0b4c2cb4173c49f2ce76f4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1732371973962 2024-11-23T14:26:17,429 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/A/e700eca174de47799af9363037b2cb4c as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/e700eca174de47799af9363037b2cb4c 2024-11-23T14:26:17,429 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting b192a9ef10554cc488020be2f9359663, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1732371974317 2024-11-23T14:26:17,430 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 51081ca63d9640a6818dd51fc1feab93, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1732371974957 2024-11-23T14:26:17,445 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in baf9478a6ce915eeb361816d4eed6489/A of baf9478a6ce915eeb361816d4eed6489 into e700eca174de47799af9363037b2cb4c(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:26:17,445 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for baf9478a6ce915eeb361816d4eed6489: 2024-11-23T14:26:17,445 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., storeName=baf9478a6ce915eeb361816d4eed6489/A, priority=13, startTime=1732371977367; duration=0sec 2024-11-23T14:26:17,445 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:26:17,445 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: baf9478a6ce915eeb361816d4eed6489:A 2024-11-23T14:26:17,450 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): baf9478a6ce915eeb361816d4eed6489#C#compaction#239 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:26:17,451 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/C/3f460f0461224924b0f1af2f4da502f9 is 50, key is test_row_0/C:col10/1732371976501/Put/seqid=0 2024-11-23T14:26:17,476 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-23T14:26:17,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:17,476 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing baf9478a6ce915eeb361816d4eed6489 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-23T14:26:17,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/A/70b7ac152ca747e38b6c96d06430aa9e is 50, key is test_row_0/A:col10/1732371976560/Put/seqid=0 2024-11-23T14:26:17,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742108_1284 (size=12325) 2024-11-23T14:26:17,505 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/C/3f460f0461224924b0f1af2f4da502f9 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/3f460f0461224924b0f1af2f4da502f9 2024-11-23T14:26:17,512 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in baf9478a6ce915eeb361816d4eed6489/C of baf9478a6ce915eeb361816d4eed6489 into 3f460f0461224924b0f1af2f4da502f9(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:26:17,512 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for baf9478a6ce915eeb361816d4eed6489: 2024-11-23T14:26:17,512 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., storeName=baf9478a6ce915eeb361816d4eed6489/C, priority=13, startTime=1732371977367; duration=0sec 2024-11-23T14:26:17,513 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:26:17,513 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: baf9478a6ce915eeb361816d4eed6489:C 2024-11-23T14:26:17,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742109_1285 (size=12151) 2024-11-23T14:26:17,536 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/A/70b7ac152ca747e38b6c96d06430aa9e 2024-11-23T14:26:17,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-23T14:26:17,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/B/536897a9c5ea4f0996e79729ceffa724 is 50, key is test_row_0/B:col10/1732371976560/Put/seqid=0 2024-11-23T14:26:17,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742110_1286 (size=12151) 2024-11-23T14:26:17,734 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. as already flushing 2024-11-23T14:26:17,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on baf9478a6ce915eeb361816d4eed6489 2024-11-23T14:26:17,780 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:17,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51138 deadline: 1732372037779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:17,783 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:17,783 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:17,783 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:17,892 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:17,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51138 deadline: 1732372037891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:17,892 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:17,892 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:17,893 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:18,008 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/B/536897a9c5ea4f0996e79729ceffa724 2024-11-23T14:26:18,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/C/b331d64733024be6b5cb7b71d18147bb is 50, key is test_row_0/C:col10/1732371976560/Put/seqid=0 2024-11-23T14:26:18,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742111_1287 (size=12151) 2024-11-23T14:26:18,059 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/C/b331d64733024be6b5cb7b71d18147bb 2024-11-23T14:26:18,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/A/70b7ac152ca747e38b6c96d06430aa9e as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/70b7ac152ca747e38b6c96d06430aa9e 2024-11-23T14:26:18,070 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/70b7ac152ca747e38b6c96d06430aa9e, entries=150, sequenceid=153, filesize=11.9 K 2024-11-23T14:26:18,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/B/536897a9c5ea4f0996e79729ceffa724 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/536897a9c5ea4f0996e79729ceffa724 2024-11-23T14:26:18,077 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/536897a9c5ea4f0996e79729ceffa724, entries=150, sequenceid=153, filesize=11.9 K 2024-11-23T14:26:18,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/C/b331d64733024be6b5cb7b71d18147bb as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/b331d64733024be6b5cb7b71d18147bb 2024-11-23T14:26:18,096 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/b331d64733024be6b5cb7b71d18147bb, entries=150, sequenceid=153, filesize=11.9 K 2024-11-23T14:26:18,097 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for baf9478a6ce915eeb361816d4eed6489 in 621ms, sequenceid=153, compaction requested=false 2024-11-23T14:26:18,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for baf9478a6ce915eeb361816d4eed6489: 2024-11-23T14:26:18,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:18,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-23T14:26:18,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-23T14:26:18,102 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-23T14:26:18,102 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6960 sec 2024-11-23T14:26:18,107 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 1.7040 sec 2024-11-23T14:26:18,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on baf9478a6ce915eeb361816d4eed6489 2024-11-23T14:26:18,109 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T14:26:18,109 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing baf9478a6ce915eeb361816d4eed6489 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-23T14:26:18,114 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/A/e42b0246f05f46d09787d379b0ebb886 is 50, key is test_row_0/A:col10/1732371978104/Put/seqid=0 2024-11-23T14:26:18,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742112_1288 (size=12151) 2024-11-23T14:26:18,218 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:18,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51138 deadline: 1732372038215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:18,219 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:18,219 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:18,219 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:18,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:18,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51138 deadline: 1732372038331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:18,334 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:18,334 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:18,334 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:18,543 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:18,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51138 deadline: 1732372038541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:18,544 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:18,544 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:18,544 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:18,545 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/A/e42b0246f05f46d09787d379b0ebb886 2024-11-23T14:26:18,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-23T14:26:18,550 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T14:26:18,551 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-23T14:26:18,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-23T14:26:18,554 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T14:26:18,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-23T14:26:18,555 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T14:26:18,555 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T14:26:18,559 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/B/8c94decc660549f9a53e5001f3e1a54b is 50, key is test_row_0/B:col10/1732371978104/Put/seqid=0 2024-11-23T14:26:18,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742113_1289 (size=12151) 2024-11-23T14:26:18,596 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/B/8c94decc660549f9a53e5001f3e1a54b 2024-11-23T14:26:18,626 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/C/fc969c2f32794359b5408a7db01e8a6c is 50, key is test_row_0/C:col10/1732371978104/Put/seqid=0 2024-11-23T14:26:18,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-23T14:26:18,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742114_1290 (size=12151) 2024-11-23T14:26:18,671 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/C/fc969c2f32794359b5408a7db01e8a6c 2024-11-23T14:26:18,677 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/A/e42b0246f05f46d09787d379b0ebb886 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/e42b0246f05f46d09787d379b0ebb886 2024-11-23T14:26:18,684 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/e42b0246f05f46d09787d379b0ebb886, entries=150, sequenceid=169, filesize=11.9 K 2024-11-23T14:26:18,686 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/B/8c94decc660549f9a53e5001f3e1a54b as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/8c94decc660549f9a53e5001f3e1a54b 2024-11-23T14:26:18,697 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/8c94decc660549f9a53e5001f3e1a54b, entries=150, sequenceid=169, filesize=11.9 K 2024-11-23T14:26:18,698 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/C/fc969c2f32794359b5408a7db01e8a6c as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/fc969c2f32794359b5408a7db01e8a6c 2024-11-23T14:26:18,706 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/fc969c2f32794359b5408a7db01e8a6c, entries=150, sequenceid=169, filesize=11.9 K 2024-11-23T14:26:18,707 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-23T14:26:18,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:18,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. as already flushing 2024-11-23T14:26:18,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:18,708 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:18,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:18,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:18,709 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for baf9478a6ce915eeb361816d4eed6489 in 600ms, sequenceid=169, compaction requested=true 2024-11-23T14:26:18,709 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for baf9478a6ce915eeb361816d4eed6489: 2024-11-23T14:26:18,710 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:26:18,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store baf9478a6ce915eeb361816d4eed6489:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T14:26:18,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:26:18,711 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:26:18,711 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36627 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:26:18,711 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1541): baf9478a6ce915eeb361816d4eed6489/A is initiating minor compaction (all files) 2024-11-23T14:26:18,711 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of baf9478a6ce915eeb361816d4eed6489/A in TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:18,711 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/e700eca174de47799af9363037b2cb4c, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/70b7ac152ca747e38b6c96d06430aa9e, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/e42b0246f05f46d09787d379b0ebb886] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp, totalSize=35.8 K 2024-11-23T14:26:18,712 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting e700eca174de47799af9363037b2cb4c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1732371974957 2024-11-23T14:26:18,712 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 70b7ac152ca747e38b6c96d06430aa9e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1732371976560 2024-11-23T14:26:18,713 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36627 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:26:18,713 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1541): baf9478a6ce915eeb361816d4eed6489/B is initiating minor compaction (all files) 2024-11-23T14:26:18,713 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of baf9478a6ce915eeb361816d4eed6489/B in TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:18,713 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/39736117dee5489cbe4147cad9cf31ea, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/536897a9c5ea4f0996e79729ceffa724, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/8c94decc660549f9a53e5001f3e1a54b] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp, totalSize=35.8 K 2024-11-23T14:26:18,713 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting e42b0246f05f46d09787d379b0ebb886, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732371977771 2024-11-23T14:26:18,714 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 39736117dee5489cbe4147cad9cf31ea, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1732371974957 2024-11-23T14:26:18,716 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 536897a9c5ea4f0996e79729ceffa724, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1732371976560 2024-11-23T14:26:18,717 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store baf9478a6ce915eeb361816d4eed6489:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T14:26:18,717 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c94decc660549f9a53e5001f3e1a54b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732371977771 2024-11-23T14:26:18,718 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:26:18,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store baf9478a6ce915eeb361816d4eed6489:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T14:26:18,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:26:18,726 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): baf9478a6ce915eeb361816d4eed6489#B#compaction#246 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:26:18,737 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/B/86e4060b7deb494cbc609c57569e90d7 is 50, key is test_row_0/B:col10/1732371978104/Put/seqid=0 2024-11-23T14:26:18,739 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): baf9478a6ce915eeb361816d4eed6489#A#compaction#247 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:26:18,739 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/A/a58cda7a68aa49b3ac2147367457446e is 50, key is test_row_0/A:col10/1732371978104/Put/seqid=0 2024-11-23T14:26:18,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742115_1291 (size=12527) 2024-11-23T14:26:18,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742116_1292 (size=12527) 2024-11-23T14:26:18,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on baf9478a6ce915eeb361816d4eed6489 2024-11-23T14:26:18,853 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T14:26:18,853 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing baf9478a6ce915eeb361816d4eed6489 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-23T14:26:18,857 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/A/32a3c3998e454e8dab565ea5ebe9488a is 50, key is test_row_0/A:col10/1732371978212/Put/seqid=0 2024-11-23T14:26:18,861 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-23T14:26:18,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:18,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. as already flushing 2024-11-23T14:26:18,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:18,862 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:18,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:18,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:18,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-23T14:26:18,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742117_1293 (size=14541) 2024-11-23T14:26:18,879 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/A/32a3c3998e454e8dab565ea5ebe9488a 2024-11-23T14:26:18,886 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/B/2e74f5a21cf64fba9d211041023c7fde is 50, key is test_row_0/B:col10/1732371978212/Put/seqid=0 2024-11-23T14:26:18,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742118_1294 (size=12151) 2024-11-23T14:26:18,916 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:18,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51138 deadline: 1732372038912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:18,917 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:18,917 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:18,917 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:19,015 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-23T14:26:19,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:19,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. as already flushing 2024-11-23T14:26:19,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:19,015 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:19,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:19,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:19,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:19,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51138 deadline: 1732372039020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:19,024 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:19,024 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:19,024 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:19,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:19,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51208 deadline: 1732372039140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:19,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:19,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51122 deadline: 1732372039140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:19,145 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to c6f7b1897123,35385,1732371902767 for 'test_row_1' in baf9478a6ce915eeb361816d4eed6489 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-23T14:26:19,146 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:19,146 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:19,146 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:19,146 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to c6f7b1897123,35385,1732371902767 for 'test_row_1' in baf9478a6ce915eeb361816d4eed6489 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-23T14:26:19,147 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:19,147 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:19,147 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:19,153 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:19,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1732372039150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:19,153 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to c6f7b1897123,35385,1732371902767 for 'test_row_0' in baf9478a6ce915eeb361816d4eed6489 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-23T14:26:19,154 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:19,154 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:19,154 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:19,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:19,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51170 deadline: 1732372039161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:19,165 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to c6f7b1897123,35385,1732371902767 for 'test_row_1' in baf9478a6ce915eeb361816d4eed6489 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-23T14:26:19,166 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:19,166 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:19,166 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:19,166 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/B/86e4060b7deb494cbc609c57569e90d7 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/86e4060b7deb494cbc609c57569e90d7 2024-11-23T14:26:19,166 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/A/a58cda7a68aa49b3ac2147367457446e as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/a58cda7a68aa49b3ac2147367457446e 2024-11-23T14:26:19,168 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-23T14:26:19,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:19,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. as already flushing 2024-11-23T14:26:19,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:19,169 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:19,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:19,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:19,177 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in baf9478a6ce915eeb361816d4eed6489/B of baf9478a6ce915eeb361816d4eed6489 into 86e4060b7deb494cbc609c57569e90d7(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:26:19,177 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for baf9478a6ce915eeb361816d4eed6489: 2024-11-23T14:26:19,177 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., storeName=baf9478a6ce915eeb361816d4eed6489/B, priority=13, startTime=1732371978711; duration=0sec 2024-11-23T14:26:19,177 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:26:19,177 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: baf9478a6ce915eeb361816d4eed6489:B 2024-11-23T14:26:19,177 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:26:19,177 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in baf9478a6ce915eeb361816d4eed6489/A of baf9478a6ce915eeb361816d4eed6489 into a58cda7a68aa49b3ac2147367457446e(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:26:19,177 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for baf9478a6ce915eeb361816d4eed6489: 2024-11-23T14:26:19,178 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., storeName=baf9478a6ce915eeb361816d4eed6489/A, priority=13, startTime=1732371978709; duration=0sec 2024-11-23T14:26:19,178 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:26:19,178 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: baf9478a6ce915eeb361816d4eed6489:A 2024-11-23T14:26:19,180 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36627 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:26:19,180 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1541): baf9478a6ce915eeb361816d4eed6489/C is initiating minor compaction (all files) 2024-11-23T14:26:19,180 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of baf9478a6ce915eeb361816d4eed6489/C in TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:19,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-23T14:26:19,181 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/3f460f0461224924b0f1af2f4da502f9, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/b331d64733024be6b5cb7b71d18147bb, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/fc969c2f32794359b5408a7db01e8a6c] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp, totalSize=35.8 K 2024-11-23T14:26:19,181 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f460f0461224924b0f1af2f4da502f9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1732371974957 2024-11-23T14:26:19,182 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting b331d64733024be6b5cb7b71d18147bb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1732371976560 2024-11-23T14:26:19,182 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting fc969c2f32794359b5408a7db01e8a6c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732371977771 2024-11-23T14:26:19,189 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): baf9478a6ce915eeb361816d4eed6489#C#compaction#250 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:26:19,190 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/C/9d1b8bfc9c2e4b98b4f20a47eb55b4da is 50, key is test_row_0/C:col10/1732371978104/Put/seqid=0 2024-11-23T14:26:19,231 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:19,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51138 deadline: 1732372039231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:19,232 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:19,232 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:19,232 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:19,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742119_1295 (size=12527) 2024-11-23T14:26:19,241 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/C/9d1b8bfc9c2e4b98b4f20a47eb55b4da as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/9d1b8bfc9c2e4b98b4f20a47eb55b4da 2024-11-23T14:26:19,249 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in baf9478a6ce915eeb361816d4eed6489/C of baf9478a6ce915eeb361816d4eed6489 into 9d1b8bfc9c2e4b98b4f20a47eb55b4da(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:26:19,249 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for baf9478a6ce915eeb361816d4eed6489: 2024-11-23T14:26:19,249 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., storeName=baf9478a6ce915eeb361816d4eed6489/C, priority=13, startTime=1732371978719; duration=0sec 2024-11-23T14:26:19,249 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:26:19,249 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: baf9478a6ce915eeb361816d4eed6489:C 2024-11-23T14:26:19,292 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/B/2e74f5a21cf64fba9d211041023c7fde 2024-11-23T14:26:19,300 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/C/6b60e57b6ef448d9840f17c3fe339794 is 50, key is test_row_0/C:col10/1732371978212/Put/seqid=0 2024-11-23T14:26:19,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742120_1296 (size=12151) 2024-11-23T14:26:19,316 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/C/6b60e57b6ef448d9840f17c3fe339794 2024-11-23T14:26:19,321 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/A/32a3c3998e454e8dab565ea5ebe9488a as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/32a3c3998e454e8dab565ea5ebe9488a 2024-11-23T14:26:19,321 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-23T14:26:19,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:19,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. as already flushing 2024-11-23T14:26:19,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:19,322 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:19,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:19,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:19,335 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/32a3c3998e454e8dab565ea5ebe9488a, entries=200, sequenceid=191, filesize=14.2 K 2024-11-23T14:26:19,336 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/B/2e74f5a21cf64fba9d211041023c7fde as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/2e74f5a21cf64fba9d211041023c7fde 2024-11-23T14:26:19,342 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/2e74f5a21cf64fba9d211041023c7fde, entries=150, sequenceid=191, filesize=11.9 K 2024-11-23T14:26:19,343 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/C/6b60e57b6ef448d9840f17c3fe339794 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/6b60e57b6ef448d9840f17c3fe339794 2024-11-23T14:26:19,347 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/6b60e57b6ef448d9840f17c3fe339794, entries=150, sequenceid=191, filesize=11.9 K 2024-11-23T14:26:19,347 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for baf9478a6ce915eeb361816d4eed6489 in 494ms, sequenceid=191, compaction requested=false 2024-11-23T14:26:19,348 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for baf9478a6ce915eeb361816d4eed6489: 2024-11-23T14:26:19,475 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-23T14:26:19,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:19,475 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing baf9478a6ce915eeb361816d4eed6489 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-23T14:26:19,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/A/bd558a1438c1406f860382db8a9da04d is 50, key is test_row_0/A:col10/1732371978894/Put/seqid=0 2024-11-23T14:26:19,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742121_1297 (size=12151) 2024-11-23T14:26:19,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on baf9478a6ce915eeb361816d4eed6489 2024-11-23T14:26:19,542 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. as already flushing 2024-11-23T14:26:19,615 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:19,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51138 deadline: 1732372039613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:19,616 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:19,616 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:19,616 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:19,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-23T14:26:19,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:19,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51138 deadline: 1732372039720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:19,722 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:19,722 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:19,722 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:19,887 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/A/bd558a1438c1406f860382db8a9da04d 2024-11-23T14:26:19,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/B/649f81df464e41d5a98e3947a37b89ba is 50, key is test_row_0/B:col10/1732371978894/Put/seqid=0 2024-11-23T14:26:19,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742122_1298 (size=12151) 2024-11-23T14:26:19,933 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:19,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51138 deadline: 1732372039930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:19,934 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:19,934 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:19,934 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:20,242 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:20,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51138 deadline: 1732372040240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:20,243 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:20,243 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:20,243 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:20,300 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/B/649f81df464e41d5a98e3947a37b89ba 2024-11-23T14:26:20,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/C/87d64a12054b44a7bd29db1c983ea283 is 50, key is test_row_0/C:col10/1732371978894/Put/seqid=0 2024-11-23T14:26:20,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742123_1299 (size=12151) 2024-11-23T14:26:20,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-23T14:26:20,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:20,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51138 deadline: 1732372040750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:20,753 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:20,753 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:20,753 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:20,766 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/C/87d64a12054b44a7bd29db1c983ea283 2024-11-23T14:26:20,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/A/bd558a1438c1406f860382db8a9da04d as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/bd558a1438c1406f860382db8a9da04d 2024-11-23T14:26:20,774 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/bd558a1438c1406f860382db8a9da04d, entries=150, sequenceid=208, filesize=11.9 K 2024-11-23T14:26:20,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/B/649f81df464e41d5a98e3947a37b89ba as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/649f81df464e41d5a98e3947a37b89ba 2024-11-23T14:26:20,780 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/649f81df464e41d5a98e3947a37b89ba, entries=150, sequenceid=208, filesize=11.9 K 2024-11-23T14:26:20,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/C/87d64a12054b44a7bd29db1c983ea283 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/87d64a12054b44a7bd29db1c983ea283 2024-11-23T14:26:20,790 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/87d64a12054b44a7bd29db1c983ea283, entries=150, sequenceid=208, filesize=11.9 K 2024-11-23T14:26:20,791 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for baf9478a6ce915eeb361816d4eed6489 in 1316ms, sequenceid=208, compaction requested=true 2024-11-23T14:26:20,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for baf9478a6ce915eeb361816d4eed6489: 2024-11-23T14:26:20,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:20,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-23T14:26:20,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-23T14:26:20,795 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-23T14:26:20,795 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2380 sec 2024-11-23T14:26:20,797 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 2.2440 sec 2024-11-23T14:26:21,774 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T14:26:21,774 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing baf9478a6ce915eeb361816d4eed6489 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-23T14:26:21,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on baf9478a6ce915eeb361816d4eed6489 2024-11-23T14:26:21,790 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/A/f6ea0e5866d646f78355422ba85cd217 is 50, key is test_row_0/A:col10/1732371979612/Put/seqid=0 2024-11-23T14:26:21,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742124_1300 (size=12151) 2024-11-23T14:26:21,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:21,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51138 deadline: 1732372041836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:21,841 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:21,842 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:21,842 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:21,953 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:21,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51138 deadline: 1732372041950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:21,954 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:21,954 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:21,954 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:22,163 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:22,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51138 deadline: 1732372042160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:22,163 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:22,163 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:22,163 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:22,201 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/A/f6ea0e5866d646f78355422ba85cd217 2024-11-23T14:26:22,208 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/B/2c1968ed8a8c4b1faabaeea718411f04 is 50, key is test_row_0/B:col10/1732371979612/Put/seqid=0 2024-11-23T14:26:22,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742125_1301 (size=12151) 2024-11-23T14:26:22,473 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:22,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51138 deadline: 1732372042470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:22,474 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:22,474 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:22,474 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:22,625 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/B/2c1968ed8a8c4b1faabaeea718411f04 2024-11-23T14:26:22,632 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/C/8b36c9f3e20144049115d90a3a0b4e73 is 50, key is test_row_0/C:col10/1732371979612/Put/seqid=0 2024-11-23T14:26:22,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742126_1302 (size=12151) 2024-11-23T14:26:22,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-23T14:26:22,721 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T14:26:22,721 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-23T14:26:22,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-23T14:26:22,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-23T14:26:22,723 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T14:26:22,723 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T14:26:22,724 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T14:26:22,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-23T14:26:22,876 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-23T14:26:22,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:22,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. as already flushing 2024-11-23T14:26:22,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:22,876 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:22,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:22,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:22,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:22,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51138 deadline: 1732372042980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:22,983 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:22,983 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:22,983 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:23,028 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-23T14:26:23,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:23,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. as already flushing 2024-11-23T14:26:23,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:23,029 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:23,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:23,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:23,036 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/C/8b36c9f3e20144049115d90a3a0b4e73 2024-11-23T14:26:23,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-23T14:26:23,041 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/A/f6ea0e5866d646f78355422ba85cd217 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/f6ea0e5866d646f78355422ba85cd217 2024-11-23T14:26:23,046 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/f6ea0e5866d646f78355422ba85cd217, entries=150, sequenceid=231, filesize=11.9 K 2024-11-23T14:26:23,047 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/B/2c1968ed8a8c4b1faabaeea718411f04 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/2c1968ed8a8c4b1faabaeea718411f04 2024-11-23T14:26:23,051 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/2c1968ed8a8c4b1faabaeea718411f04, entries=150, sequenceid=231, filesize=11.9 K 2024-11-23T14:26:23,051 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/C/8b36c9f3e20144049115d90a3a0b4e73 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/8b36c9f3e20144049115d90a3a0b4e73 2024-11-23T14:26:23,055 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/8b36c9f3e20144049115d90a3a0b4e73, entries=150, sequenceid=231, filesize=11.9 K 2024-11-23T14:26:23,056 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for baf9478a6ce915eeb361816d4eed6489 in 1282ms, sequenceid=231, compaction requested=true 2024-11-23T14:26:23,056 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for baf9478a6ce915eeb361816d4eed6489: 2024-11-23T14:26:23,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store baf9478a6ce915eeb361816d4eed6489:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T14:26:23,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:26:23,056 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T14:26:23,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store baf9478a6ce915eeb361816d4eed6489:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T14:26:23,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:26:23,056 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T14:26:23,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store baf9478a6ce915eeb361816d4eed6489:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T14:26:23,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:26:23,057 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48980 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T14:26:23,057 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1541): baf9478a6ce915eeb361816d4eed6489/B is initiating minor compaction (all files) 2024-11-23T14:26:23,057 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of baf9478a6ce915eeb361816d4eed6489/B in TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:23,057 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51370 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T14:26:23,057 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/86e4060b7deb494cbc609c57569e90d7, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/2e74f5a21cf64fba9d211041023c7fde, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/649f81df464e41d5a98e3947a37b89ba, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/2c1968ed8a8c4b1faabaeea718411f04] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp, totalSize=47.8 K 2024-11-23T14:26:23,057 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1541): baf9478a6ce915eeb361816d4eed6489/A is initiating minor compaction (all files) 2024-11-23T14:26:23,057 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of baf9478a6ce915eeb361816d4eed6489/A in TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:23,058 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/a58cda7a68aa49b3ac2147367457446e, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/32a3c3998e454e8dab565ea5ebe9488a, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/bd558a1438c1406f860382db8a9da04d, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/f6ea0e5866d646f78355422ba85cd217] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp, totalSize=50.2 K 2024-11-23T14:26:23,058 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 86e4060b7deb494cbc609c57569e90d7, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732371977771 2024-11-23T14:26:23,058 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting a58cda7a68aa49b3ac2147367457446e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732371977771 2024-11-23T14:26:23,058 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 32a3c3998e454e8dab565ea5ebe9488a, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1732371978158 2024-11-23T14:26:23,058 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e74f5a21cf64fba9d211041023c7fde, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1732371978206 2024-11-23T14:26:23,058 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 649f81df464e41d5a98e3947a37b89ba, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732371978880 2024-11-23T14:26:23,059 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd558a1438c1406f860382db8a9da04d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732371978880 2024-11-23T14:26:23,059 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c1968ed8a8c4b1faabaeea718411f04, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1732371979607 2024-11-23T14:26:23,059 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting f6ea0e5866d646f78355422ba85cd217, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1732371979607 2024-11-23T14:26:23,067 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): baf9478a6ce915eeb361816d4eed6489#B#compaction#258 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:26:23,067 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/B/bf26281ba7804e9b87af66ee909a0f61 is 50, key is test_row_0/B:col10/1732371979612/Put/seqid=0 2024-11-23T14:26:23,075 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): baf9478a6ce915eeb361816d4eed6489#A#compaction#259 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:26:23,075 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/A/dce7a048321c4a7a856c5f37d076a929 is 50, key is test_row_0/A:col10/1732371979612/Put/seqid=0 2024-11-23T14:26:23,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742127_1303 (size=12663) 2024-11-23T14:26:23,089 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/B/bf26281ba7804e9b87af66ee909a0f61 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/bf26281ba7804e9b87af66ee909a0f61 2024-11-23T14:26:23,094 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in baf9478a6ce915eeb361816d4eed6489/B of baf9478a6ce915eeb361816d4eed6489 into bf26281ba7804e9b87af66ee909a0f61(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:26:23,094 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for baf9478a6ce915eeb361816d4eed6489: 2024-11-23T14:26:23,094 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., storeName=baf9478a6ce915eeb361816d4eed6489/B, priority=12, startTime=1732371983056; duration=0sec 2024-11-23T14:26:23,094 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:26:23,094 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: baf9478a6ce915eeb361816d4eed6489:B 2024-11-23T14:26:23,094 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T14:26:23,095 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48980 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T14:26:23,095 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1541): baf9478a6ce915eeb361816d4eed6489/C is initiating minor compaction (all files) 2024-11-23T14:26:23,095 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of baf9478a6ce915eeb361816d4eed6489/C in TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:23,095 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/9d1b8bfc9c2e4b98b4f20a47eb55b4da, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/6b60e57b6ef448d9840f17c3fe339794, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/87d64a12054b44a7bd29db1c983ea283, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/8b36c9f3e20144049115d90a3a0b4e73] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp, totalSize=47.8 K 2024-11-23T14:26:23,096 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d1b8bfc9c2e4b98b4f20a47eb55b4da, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732371977771 2024-11-23T14:26:23,096 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b60e57b6ef448d9840f17c3fe339794, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1732371978206 2024-11-23T14:26:23,096 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 87d64a12054b44a7bd29db1c983ea283, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732371978880 2024-11-23T14:26:23,096 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b36c9f3e20144049115d90a3a0b4e73, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1732371979607 2024-11-23T14:26:23,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742128_1304 (size=12663) 2024-11-23T14:26:23,106 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): baf9478a6ce915eeb361816d4eed6489#C#compaction#260 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:26:23,106 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/C/915f1ab37e9d48098ea33b1469298c95 is 50, key is test_row_0/C:col10/1732371979612/Put/seqid=0 2024-11-23T14:26:23,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742129_1305 (size=12663) 2024-11-23T14:26:23,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on baf9478a6ce915eeb361816d4eed6489 2024-11-23T14:26:23,162 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T14:26:23,162 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing baf9478a6ce915eeb361816d4eed6489 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-23T14:26:23,181 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-23T14:26:23,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:23,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. as already flushing 2024-11-23T14:26:23,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:23,182 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:23,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:23,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:23,187 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/A/fd1cc12f013540d590b0679f37f345e4 is 50, key is test_row_0/A:col10/1732371981834/Put/seqid=0 2024-11-23T14:26:23,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742130_1306 (size=14541) 2024-11-23T14:26:23,197 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/A/fd1cc12f013540d590b0679f37f345e4 2024-11-23T14:26:23,204 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/B/2bb48d0b07944a3db8f50ac704bd91b1 is 50, key is test_row_0/B:col10/1732371981834/Put/seqid=0 2024-11-23T14:26:23,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:23,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742131_1307 (size=12151) 2024-11-23T14:26:23,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51122 deadline: 1732372043203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:23,208 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:23,208 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:23,208 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:23,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1732372043204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:23,208 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:23,208 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/B/2bb48d0b07944a3db8f50ac704bd91b1 2024-11-23T14:26:23,208 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:23,208 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:23,208 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:23,208 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:23,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51208 deadline: 1732372043205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:23,209 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:23,209 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:23,209 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:23,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:23,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51170 deadline: 1732372043210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:23,213 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to c6f7b1897123,35385,1732371902767 for 'test_row_1' in baf9478a6ce915eeb361816d4eed6489 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8242 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-23T14:26:23,213 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:23,213 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:23,213 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:23,215 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/C/4876f3251ea8498a997159e32135152a is 50, key is test_row_0/C:col10/1732371981834/Put/seqid=0 2024-11-23T14:26:23,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742132_1308 (size=12151) 2024-11-23T14:26:23,219 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/C/4876f3251ea8498a997159e32135152a 2024-11-23T14:26:23,223 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/A/fd1cc12f013540d590b0679f37f345e4 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/fd1cc12f013540d590b0679f37f345e4 2024-11-23T14:26:23,227 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/fd1cc12f013540d590b0679f37f345e4, entries=200, sequenceid=246, filesize=14.2 K 2024-11-23T14:26:23,228 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/B/2bb48d0b07944a3db8f50ac704bd91b1 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/2bb48d0b07944a3db8f50ac704bd91b1 2024-11-23T14:26:23,232 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/2bb48d0b07944a3db8f50ac704bd91b1, entries=150, sequenceid=246, filesize=11.9 K 2024-11-23T14:26:23,232 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/C/4876f3251ea8498a997159e32135152a as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/4876f3251ea8498a997159e32135152a 2024-11-23T14:26:23,237 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/4876f3251ea8498a997159e32135152a, entries=150, sequenceid=246, filesize=11.9 K 2024-11-23T14:26:23,238 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for baf9478a6ce915eeb361816d4eed6489 in 76ms, sequenceid=246, compaction requested=false 2024-11-23T14:26:23,238 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for baf9478a6ce915eeb361816d4eed6489: 2024-11-23T14:26:23,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on baf9478a6ce915eeb361816d4eed6489 2024-11-23T14:26:23,313 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T14:26:23,313 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing baf9478a6ce915eeb361816d4eed6489 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-23T14:26:23,319 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/A/2c25ab3e7d364e1182681d2354fda98b is 50, key is test_row_0/A:col10/1732371983203/Put/seqid=0 2024-11-23T14:26:23,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742133_1309 (size=12301) 2024-11-23T14:26:23,334 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-23T14:26:23,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:23,334 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:23,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. as already flushing 2024-11-23T14:26:23,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:23,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1732372043330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:23,334 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:23,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:23,334 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:23,334 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:23,334 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:23,335 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:23,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51122 deadline: 1732372043330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:23,335 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:23,335 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:23,335 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:23,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:23,337 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:23,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51208 deadline: 1732372043334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:23,337 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:23,337 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:23,337 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:23,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-23T14:26:23,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:23,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1732372043440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:23,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:23,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:23,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:23,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51122 deadline: 1732372043440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:23,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:23,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:23,443 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:23,443 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:23,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51208 deadline: 1732372043440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:23,443 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:23,443 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:23,444 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:23,444 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:23,486 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-23T14:26:23,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:23,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. as already flushing 2024-11-23T14:26:23,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:23,487 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:23,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:23,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:23,506 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/A/dce7a048321c4a7a856c5f37d076a929 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/dce7a048321c4a7a856c5f37d076a929 2024-11-23T14:26:23,510 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in baf9478a6ce915eeb361816d4eed6489/A of baf9478a6ce915eeb361816d4eed6489 into dce7a048321c4a7a856c5f37d076a929(size=12.4 K), total size for store is 26.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:26:23,510 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for baf9478a6ce915eeb361816d4eed6489: 2024-11-23T14:26:23,510 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., storeName=baf9478a6ce915eeb361816d4eed6489/A, priority=12, startTime=1732371983056; duration=0sec 2024-11-23T14:26:23,510 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:26:23,510 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: baf9478a6ce915eeb361816d4eed6489:A 2024-11-23T14:26:23,527 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/C/915f1ab37e9d48098ea33b1469298c95 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/915f1ab37e9d48098ea33b1469298c95 2024-11-23T14:26:23,531 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in baf9478a6ce915eeb361816d4eed6489/C of baf9478a6ce915eeb361816d4eed6489 into 915f1ab37e9d48098ea33b1469298c95(size=12.4 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:26:23,531 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for baf9478a6ce915eeb361816d4eed6489: 2024-11-23T14:26:23,531 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., storeName=baf9478a6ce915eeb361816d4eed6489/C, priority=12, startTime=1732371983056; duration=0sec 2024-11-23T14:26:23,531 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:26:23,531 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: baf9478a6ce915eeb361816d4eed6489:C 2024-11-23T14:26:23,639 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-23T14:26:23,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:23,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. as already flushing 2024-11-23T14:26:23,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:23,639 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:23,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:23,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:23,653 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:23,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1732372043650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:23,653 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:23,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51122 deadline: 1732372043650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:23,653 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:23,653 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:23,653 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:23,653 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:23,653 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:23,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51208 deadline: 1732372043650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:23,653 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:23,653 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:23,653 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:23,653 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:23,654 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:23,730 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/A/2c25ab3e7d364e1182681d2354fda98b 2024-11-23T14:26:23,736 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/B/c4f57c4c586a4b97b7de50b1c36a96ec is 50, key is test_row_0/B:col10/1732371983203/Put/seqid=0 2024-11-23T14:26:23,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742134_1310 (size=12301) 2024-11-23T14:26:23,792 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-23T14:26:23,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:23,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. as already flushing 2024-11-23T14:26:23,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:23,792 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:23,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:23,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:23,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-23T14:26:23,945 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-23T14:26:23,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:23,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. as already flushing 2024-11-23T14:26:23,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:23,945 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:23,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:23,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:23,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:23,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1732372043961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:23,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:23,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:23,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:23,965 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:23,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51208 deadline: 1732372043961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:23,966 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:23,966 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:23,966 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:23,966 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:23,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51122 deadline: 1732372043961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:23,967 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:23,967 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:23,967 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:23,992 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:23,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51138 deadline: 1732372043990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:23,993 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to c6f7b1897123,35385,1732371902767 for 'test_row_0' in baf9478a6ce915eeb361816d4eed6489 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-23T14:26:23,993 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:23,993 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:23,993 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:24,101 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-23T14:26:24,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:24,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. as already flushing 2024-11-23T14:26:24,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:24,102 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:24,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:24,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:24,140 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/B/c4f57c4c586a4b97b7de50b1c36a96ec 2024-11-23T14:26:24,146 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/C/3dded0da65aa4ad9a2a4b05b24455f4b is 50, key is test_row_0/C:col10/1732371983203/Put/seqid=0 2024-11-23T14:26:24,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742135_1311 (size=12301) 2024-11-23T14:26:24,255 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-23T14:26:24,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:24,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. as already flushing 2024-11-23T14:26:24,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:24,255 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:24,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:24,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:24,408 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-23T14:26:24,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:24,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. as already flushing 2024-11-23T14:26:24,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:24,408 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:24,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:24,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:24,472 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:24,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1732372044470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:24,472 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:24,472 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:24,472 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:24,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:24,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51208 deadline: 1732372044470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:24,473 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:24,473 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:24,473 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:24,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:24,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51122 deadline: 1732372044480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:24,483 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:24,483 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:24,483 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:24,550 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/C/3dded0da65aa4ad9a2a4b05b24455f4b 2024-11-23T14:26:24,556 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/A/2c25ab3e7d364e1182681d2354fda98b as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/2c25ab3e7d364e1182681d2354fda98b 2024-11-23T14:26:24,561 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/2c25ab3e7d364e1182681d2354fda98b, entries=150, sequenceid=270, filesize=12.0 K 2024-11-23T14:26:24,562 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-23T14:26:24,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:24,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. as already flushing 2024-11-23T14:26:24,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:24,562 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/B/c4f57c4c586a4b97b7de50b1c36a96ec as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/c4f57c4c586a4b97b7de50b1c36a96ec 2024-11-23T14:26:24,562 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:24,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:24,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:24,569 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/c4f57c4c586a4b97b7de50b1c36a96ec, entries=150, sequenceid=270, filesize=12.0 K 2024-11-23T14:26:24,570 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/C/3dded0da65aa4ad9a2a4b05b24455f4b as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/3dded0da65aa4ad9a2a4b05b24455f4b 2024-11-23T14:26:24,574 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/3dded0da65aa4ad9a2a4b05b24455f4b, entries=150, sequenceid=270, filesize=12.0 K 2024-11-23T14:26:24,576 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for baf9478a6ce915eeb361816d4eed6489 in 1263ms, sequenceid=270, compaction requested=true 2024-11-23T14:26:24,576 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for baf9478a6ce915eeb361816d4eed6489: 2024-11-23T14:26:24,577 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:26:24,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store baf9478a6ce915eeb361816d4eed6489:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T14:26:24,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:26:24,578 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:26:24,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store baf9478a6ce915eeb361816d4eed6489:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T14:26:24,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:26:24,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store baf9478a6ce915eeb361816d4eed6489:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T14:26:24,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:26:24,580 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39505 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:26:24,580 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1541): baf9478a6ce915eeb361816d4eed6489/A is initiating minor compaction (all files) 2024-11-23T14:26:24,580 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of baf9478a6ce915eeb361816d4eed6489/A in TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:24,580 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/dce7a048321c4a7a856c5f37d076a929, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/fd1cc12f013540d590b0679f37f345e4, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/2c25ab3e7d364e1182681d2354fda98b] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp, totalSize=38.6 K 2024-11-23T14:26:24,581 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:26:24,581 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting dce7a048321c4a7a856c5f37d076a929, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1732371979607 2024-11-23T14:26:24,581 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1541): baf9478a6ce915eeb361816d4eed6489/B is initiating minor compaction (all files) 2024-11-23T14:26:24,581 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of baf9478a6ce915eeb361816d4eed6489/B in TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:24,581 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/bf26281ba7804e9b87af66ee909a0f61, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/2bb48d0b07944a3db8f50ac704bd91b1, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/c4f57c4c586a4b97b7de50b1c36a96ec] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp, totalSize=36.2 K 2024-11-23T14:26:24,583 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd1cc12f013540d590b0679f37f345e4, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1732371981812 2024-11-23T14:26:24,583 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting bf26281ba7804e9b87af66ee909a0f61, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1732371979607 2024-11-23T14:26:24,583 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 2bb48d0b07944a3db8f50ac704bd91b1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1732371981812 2024-11-23T14:26:24,583 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2c25ab3e7d364e1182681d2354fda98b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1732371983203 2024-11-23T14:26:24,584 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting c4f57c4c586a4b97b7de50b1c36a96ec, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1732371983203 2024-11-23T14:26:24,603 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): baf9478a6ce915eeb361816d4eed6489#B#compaction#267 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:26:24,604 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/B/8557925f7f3a4204b4d48da94cd683f1 is 50, key is test_row_0/B:col10/1732371983203/Put/seqid=0 2024-11-23T14:26:24,615 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): baf9478a6ce915eeb361816d4eed6489#A#compaction#268 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:26:24,615 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/A/07de77a219ab4cc98fab9491ab4608e9 is 50, key is test_row_0/A:col10/1732371983203/Put/seqid=0 2024-11-23T14:26:24,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742136_1312 (size=12915) 2024-11-23T14:26:24,658 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/B/8557925f7f3a4204b4d48da94cd683f1 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/8557925f7f3a4204b4d48da94cd683f1 2024-11-23T14:26:24,665 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in baf9478a6ce915eeb361816d4eed6489/B of baf9478a6ce915eeb361816d4eed6489 into 8557925f7f3a4204b4d48da94cd683f1(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:26:24,665 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for baf9478a6ce915eeb361816d4eed6489: 2024-11-23T14:26:24,665 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., storeName=baf9478a6ce915eeb361816d4eed6489/B, priority=13, startTime=1732371984578; duration=0sec 2024-11-23T14:26:24,665 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:26:24,665 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: baf9478a6ce915eeb361816d4eed6489:B 2024-11-23T14:26:24,665 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:26:24,667 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:26:24,667 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1541): baf9478a6ce915eeb361816d4eed6489/C is initiating minor compaction (all files) 2024-11-23T14:26:24,667 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of baf9478a6ce915eeb361816d4eed6489/C in TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:24,667 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/915f1ab37e9d48098ea33b1469298c95, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/4876f3251ea8498a997159e32135152a, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/3dded0da65aa4ad9a2a4b05b24455f4b] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp, totalSize=36.2 K 2024-11-23T14:26:24,668 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 915f1ab37e9d48098ea33b1469298c95, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1732371979607 2024-11-23T14:26:24,669 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 4876f3251ea8498a997159e32135152a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1732371981812 2024-11-23T14:26:24,669 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 3dded0da65aa4ad9a2a4b05b24455f4b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1732371983203 2024-11-23T14:26:24,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742137_1313 (size=12915) 2024-11-23T14:26:24,682 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/A/07de77a219ab4cc98fab9491ab4608e9 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/07de77a219ab4cc98fab9491ab4608e9 2024-11-23T14:26:24,690 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): baf9478a6ce915eeb361816d4eed6489#C#compaction#269 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:26:24,690 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in baf9478a6ce915eeb361816d4eed6489/A of baf9478a6ce915eeb361816d4eed6489 into 07de77a219ab4cc98fab9491ab4608e9(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:26:24,691 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for baf9478a6ce915eeb361816d4eed6489: 2024-11-23T14:26:24,691 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/C/bf2e3d352710479e8353f1f9a2ce3206 is 50, key is test_row_0/C:col10/1732371983203/Put/seqid=0 2024-11-23T14:26:24,691 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., storeName=baf9478a6ce915eeb361816d4eed6489/A, priority=13, startTime=1732371984576; duration=0sec 2024-11-23T14:26:24,691 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:26:24,691 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: baf9478a6ce915eeb361816d4eed6489:A 2024-11-23T14:26:24,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742138_1314 (size=12915) 2024-11-23T14:26:24,711 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/C/bf2e3d352710479e8353f1f9a2ce3206 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/bf2e3d352710479e8353f1f9a2ce3206 2024-11-23T14:26:24,716 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-23T14:26:24,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:24,717 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing baf9478a6ce915eeb361816d4eed6489 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-23T14:26:24,718 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in baf9478a6ce915eeb361816d4eed6489/C of baf9478a6ce915eeb361816d4eed6489 into bf2e3d352710479e8353f1f9a2ce3206(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:26:24,719 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for baf9478a6ce915eeb361816d4eed6489: 2024-11-23T14:26:24,719 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., storeName=baf9478a6ce915eeb361816d4eed6489/C, priority=13, startTime=1732371984578; duration=0sec 2024-11-23T14:26:24,719 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:26:24,719 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: baf9478a6ce915eeb361816d4eed6489:C 2024-11-23T14:26:24,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/A/72dfd8ea3cd2444bb73d8e57f4eebd50 is 50, key is test_row_0/A:col10/1732371983328/Put/seqid=0 2024-11-23T14:26:24,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742139_1315 (size=12301) 2024-11-23T14:26:24,747 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/A/72dfd8ea3cd2444bb73d8e57f4eebd50 2024-11-23T14:26:24,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/B/f95f96fc0cb34b4ea36e9beb96cf8c69 is 50, key is test_row_0/B:col10/1732371983328/Put/seqid=0 2024-11-23T14:26:24,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742140_1316 (size=12301) 2024-11-23T14:26:24,765 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/B/f95f96fc0cb34b4ea36e9beb96cf8c69 2024-11-23T14:26:24,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/C/a40e8a6d3aba45cc8ee8a4c18a8a675c is 50, key is test_row_0/C:col10/1732371983328/Put/seqid=0 2024-11-23T14:26:24,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742141_1317 (size=12301) 2024-11-23T14:26:24,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-23T14:26:25,187 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/C/a40e8a6d3aba45cc8ee8a4c18a8a675c 2024-11-23T14:26:25,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/A/72dfd8ea3cd2444bb73d8e57f4eebd50 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/72dfd8ea3cd2444bb73d8e57f4eebd50 2024-11-23T14:26:25,211 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/72dfd8ea3cd2444bb73d8e57f4eebd50, entries=150, sequenceid=286, filesize=12.0 K 2024-11-23T14:26:25,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/B/f95f96fc0cb34b4ea36e9beb96cf8c69 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/f95f96fc0cb34b4ea36e9beb96cf8c69 2024-11-23T14:26:25,217 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/f95f96fc0cb34b4ea36e9beb96cf8c69, entries=150, sequenceid=286, filesize=12.0 K 2024-11-23T14:26:25,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/C/a40e8a6d3aba45cc8ee8a4c18a8a675c as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/a40e8a6d3aba45cc8ee8a4c18a8a675c 2024-11-23T14:26:25,228 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/a40e8a6d3aba45cc8ee8a4c18a8a675c, entries=150, sequenceid=286, filesize=12.0 K 2024-11-23T14:26:25,229 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=0 B/0 for baf9478a6ce915eeb361816d4eed6489 in 512ms, sequenceid=286, compaction requested=false 2024-11-23T14:26:25,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for baf9478a6ce915eeb361816d4eed6489: 2024-11-23T14:26:25,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:25,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-23T14:26:25,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-23T14:26:25,233 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-23T14:26:25,233 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5070 sec 2024-11-23T14:26:25,236 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 2.5130 sec 2024-11-23T14:26:25,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on baf9478a6ce915eeb361816d4eed6489 2024-11-23T14:26:25,497 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T14:26:25,497 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing baf9478a6ce915eeb361816d4eed6489 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-23T14:26:25,502 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/A/fceec668b5d847e3868876a38cab7f3b is 50, key is test_row_0/A:col10/1732371985491/Put/seqid=0 2024-11-23T14:26:25,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742142_1318 (size=14741) 2024-11-23T14:26:25,546 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:25,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51122 deadline: 1732372045542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:25,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:25,547 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:25,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1732372045543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:25,547 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:25,547 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:25,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:25,547 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:25,547 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:25,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51208 deadline: 1732372045543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:25,547 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:25,548 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:25,548 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:25,548 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:25,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:25,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51122 deadline: 1732372045650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:25,652 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:25,652 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:25,652 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:25,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:25,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1732372045650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:25,653 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:25,653 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:25,653 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:25,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51208 deadline: 1732372045650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:25,653 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:25,653 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:25,653 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:25,653 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:25,862 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:25,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51122 deadline: 1732372045860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:25,862 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:25,862 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:25,862 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:25,863 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:25,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51208 deadline: 1732372045860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:25,863 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:25,863 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:25,863 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:25,863 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:25,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1732372045860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:25,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:25,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:25,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:25,910 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/A/fceec668b5d847e3868876a38cab7f3b 2024-11-23T14:26:25,919 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/B/a78e270c3de541dd8b0b62208ba8538d is 50, key is test_row_0/B:col10/1732371985491/Put/seqid=0 2024-11-23T14:26:25,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742143_1319 (size=12301) 2024-11-23T14:26:25,929 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/B/a78e270c3de541dd8b0b62208ba8538d 2024-11-23T14:26:25,935 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/C/d86d07f1eea24ef8a977cb4f80207878 is 50, key is test_row_0/C:col10/1732371985491/Put/seqid=0 2024-11-23T14:26:25,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742144_1320 (size=12301) 2024-11-23T14:26:26,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:26,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51138 deadline: 1732372046000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:26,003 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to c6f7b1897123,35385,1732371902767 for 'test_row_0' in baf9478a6ce915eeb361816d4eed6489 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-23T14:26:26,003 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:26,003 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:26,003 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:26,171 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:26,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51122 deadline: 1732372046170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:26,172 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:26,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:26,172 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:26,172 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:26,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51208 deadline: 1732372046171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:26,172 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:26,172 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:26,172 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:26,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:26,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1732372046171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:26,173 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:26,173 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:26,173 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:26,340 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/C/d86d07f1eea24ef8a977cb4f80207878 2024-11-23T14:26:26,344 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/A/fceec668b5d847e3868876a38cab7f3b as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/fceec668b5d847e3868876a38cab7f3b 2024-11-23T14:26:26,358 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/fceec668b5d847e3868876a38cab7f3b, entries=200, sequenceid=299, filesize=14.4 K 2024-11-23T14:26:26,359 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/B/a78e270c3de541dd8b0b62208ba8538d as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/a78e270c3de541dd8b0b62208ba8538d 2024-11-23T14:26:26,364 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/a78e270c3de541dd8b0b62208ba8538d, entries=150, sequenceid=299, filesize=12.0 K 2024-11-23T14:26:26,365 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/C/d86d07f1eea24ef8a977cb4f80207878 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/d86d07f1eea24ef8a977cb4f80207878 2024-11-23T14:26:26,372 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/d86d07f1eea24ef8a977cb4f80207878, entries=150, sequenceid=299, filesize=12.0 K 2024-11-23T14:26:26,373 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for baf9478a6ce915eeb361816d4eed6489 in 876ms, sequenceid=299, compaction requested=true 2024-11-23T14:26:26,373 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for baf9478a6ce915eeb361816d4eed6489: 2024-11-23T14:26:26,374 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:26:26,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store baf9478a6ce915eeb361816d4eed6489:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T14:26:26,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:26:26,374 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:26:26,375 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39957 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:26:26,375 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1541): baf9478a6ce915eeb361816d4eed6489/A is initiating minor compaction (all files) 2024-11-23T14:26:26,375 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of baf9478a6ce915eeb361816d4eed6489/A in TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:26,375 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/07de77a219ab4cc98fab9491ab4608e9, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/72dfd8ea3cd2444bb73d8e57f4eebd50, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/fceec668b5d847e3868876a38cab7f3b] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp, totalSize=39.0 K 2024-11-23T14:26:26,376 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:26:26,376 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1541): baf9478a6ce915eeb361816d4eed6489/B is initiating minor compaction (all files) 2024-11-23T14:26:26,376 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of baf9478a6ce915eeb361816d4eed6489/B in TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:26,376 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/8557925f7f3a4204b4d48da94cd683f1, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/f95f96fc0cb34b4ea36e9beb96cf8c69, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/a78e270c3de541dd8b0b62208ba8538d] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp, totalSize=36.6 K 2024-11-23T14:26:26,376 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 07de77a219ab4cc98fab9491ab4608e9, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1732371983203 2024-11-23T14:26:26,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store baf9478a6ce915eeb361816d4eed6489:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T14:26:26,377 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 8557925f7f3a4204b4d48da94cd683f1, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1732371983203 2024-11-23T14:26:26,377 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 72dfd8ea3cd2444bb73d8e57f4eebd50, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1732371983328 2024-11-23T14:26:26,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:26:26,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store baf9478a6ce915eeb361816d4eed6489:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T14:26:26,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:26:26,377 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting f95f96fc0cb34b4ea36e9beb96cf8c69, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1732371983328 2024-11-23T14:26:26,377 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting fceec668b5d847e3868876a38cab7f3b, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1732371985491 2024-11-23T14:26:26,378 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting a78e270c3de541dd8b0b62208ba8538d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1732371985491 2024-11-23T14:26:26,396 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): baf9478a6ce915eeb361816d4eed6489#A#compaction#276 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:26:26,396 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/A/ada6bc81478d4a82b93f8af5469e76af is 50, key is test_row_0/A:col10/1732371985491/Put/seqid=0 2024-11-23T14:26:26,410 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): baf9478a6ce915eeb361816d4eed6489#B#compaction#277 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:26:26,412 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/B/9310faba50a04f48b4f7e8f36b6b09d7 is 50, key is test_row_0/B:col10/1732371985491/Put/seqid=0 2024-11-23T14:26:26,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742145_1321 (size=13017) 2024-11-23T14:26:26,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742146_1322 (size=13017) 2024-11-23T14:26:26,453 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/B/9310faba50a04f48b4f7e8f36b6b09d7 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/9310faba50a04f48b4f7e8f36b6b09d7 2024-11-23T14:26:26,459 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in baf9478a6ce915eeb361816d4eed6489/B of baf9478a6ce915eeb361816d4eed6489 into 9310faba50a04f48b4f7e8f36b6b09d7(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:26:26,459 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for baf9478a6ce915eeb361816d4eed6489: 2024-11-23T14:26:26,459 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., storeName=baf9478a6ce915eeb361816d4eed6489/B, priority=13, startTime=1732371986374; duration=0sec 2024-11-23T14:26:26,459 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:26:26,459 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: baf9478a6ce915eeb361816d4eed6489:B 2024-11-23T14:26:26,459 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:26:26,460 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:26:26,460 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1541): baf9478a6ce915eeb361816d4eed6489/C is initiating minor compaction (all files) 2024-11-23T14:26:26,460 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of baf9478a6ce915eeb361816d4eed6489/C in TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:26,460 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/bf2e3d352710479e8353f1f9a2ce3206, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/a40e8a6d3aba45cc8ee8a4c18a8a675c, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/d86d07f1eea24ef8a977cb4f80207878] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp, totalSize=36.6 K 2024-11-23T14:26:26,461 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting bf2e3d352710479e8353f1f9a2ce3206, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1732371983203 2024-11-23T14:26:26,461 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting a40e8a6d3aba45cc8ee8a4c18a8a675c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1732371983328 2024-11-23T14:26:26,461 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting d86d07f1eea24ef8a977cb4f80207878, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1732371985491 2024-11-23T14:26:26,475 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): baf9478a6ce915eeb361816d4eed6489#C#compaction#278 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:26:26,475 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/C/74ec07b6e99b469eb30d3692f1867ae1 is 50, key is test_row_0/C:col10/1732371985491/Put/seqid=0 2024-11-23T14:26:26,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742147_1323 (size=13017) 2024-11-23T14:26:26,527 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/C/74ec07b6e99b469eb30d3692f1867ae1 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/74ec07b6e99b469eb30d3692f1867ae1 2024-11-23T14:26:26,538 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in baf9478a6ce915eeb361816d4eed6489/C of baf9478a6ce915eeb361816d4eed6489 into 74ec07b6e99b469eb30d3692f1867ae1(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:26:26,538 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for baf9478a6ce915eeb361816d4eed6489: 2024-11-23T14:26:26,538 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., storeName=baf9478a6ce915eeb361816d4eed6489/C, priority=13, startTime=1732371986377; duration=0sec 2024-11-23T14:26:26,539 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:26:26,539 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: baf9478a6ce915eeb361816d4eed6489:C 2024-11-23T14:26:26,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on baf9478a6ce915eeb361816d4eed6489 2024-11-23T14:26:26,682 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T14:26:26,682 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing baf9478a6ce915eeb361816d4eed6489 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-23T14:26:26,695 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:26,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51122 deadline: 1732372046693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:26,694 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:26,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51208 deadline: 1732372046691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:26,696 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:26,696 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:26,696 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:26,696 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:26,696 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:26,696 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:26,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:26,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1732372046695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:26,698 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/A/1f71dac9d319448f9072834144085780 is 50, key is test_row_0/A:col10/1732371985542/Put/seqid=0 2024-11-23T14:26:26,698 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:26,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:26,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:26,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742148_1324 (size=14741) 2024-11-23T14:26:26,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:26,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51208 deadline: 1732372046800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:26,802 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:26,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:26,802 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:26,802 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:26,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51122 deadline: 1732372046800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:26,803 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:26,803 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:26,803 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:26,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:26,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1732372046811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:26,812 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:26,812 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:26,813 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:26,848 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/A/ada6bc81478d4a82b93f8af5469e76af as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/ada6bc81478d4a82b93f8af5469e76af 2024-11-23T14:26:26,856 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in baf9478a6ce915eeb361816d4eed6489/A of baf9478a6ce915eeb361816d4eed6489 into ada6bc81478d4a82b93f8af5469e76af(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:26:26,856 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for baf9478a6ce915eeb361816d4eed6489: 2024-11-23T14:26:26,856 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., storeName=baf9478a6ce915eeb361816d4eed6489/A, priority=13, startTime=1732371986374; duration=0sec 2024-11-23T14:26:26,856 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:26:26,856 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: baf9478a6ce915eeb361816d4eed6489:A 2024-11-23T14:26:26,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-23T14:26:26,891 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T14:26:26,891 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-23T14:26:26,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-23T14:26:26,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-23T14:26:26,896 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T14:26:26,898 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T14:26:26,898 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T14:26:27,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-23T14:26:27,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:27,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51208 deadline: 1732372047010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:27,011 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:27,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:27,012 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:27,012 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:27,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51122 deadline: 1732372047010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:27,012 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:27,012 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:27,012 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:27,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:27,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1732372047020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:27,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:27,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:27,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:27,055 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-23T14:26:27,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:27,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. as already flushing 2024-11-23T14:26:27,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:27,056 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:27,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:27,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:27,125 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/A/1f71dac9d319448f9072834144085780 2024-11-23T14:26:27,144 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/B/1da6857a570d43d5ae342084aba17d99 is 50, key is test_row_0/B:col10/1732371985542/Put/seqid=0 2024-11-23T14:26:27,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742149_1325 (size=12301) 2024-11-23T14:26:27,174 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/B/1da6857a570d43d5ae342084aba17d99 2024-11-23T14:26:27,182 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/C/a14182a9ee9a490fbde7bc619ddea4fb is 50, key is test_row_0/C:col10/1732371985542/Put/seqid=0 2024-11-23T14:26:27,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742150_1326 (size=12301) 2024-11-23T14:26:27,200 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/C/a14182a9ee9a490fbde7bc619ddea4fb 2024-11-23T14:26:27,206 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/A/1f71dac9d319448f9072834144085780 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/1f71dac9d319448f9072834144085780 2024-11-23T14:26:27,209 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-23T14:26:27,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:27,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. as already flushing 2024-11-23T14:26:27,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:27,210 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:27,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:27,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-23T14:26:27,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:27,213 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/1f71dac9d319448f9072834144085780, entries=200, sequenceid=328, filesize=14.4 K 2024-11-23T14:26:27,215 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/B/1da6857a570d43d5ae342084aba17d99 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/1da6857a570d43d5ae342084aba17d99 2024-11-23T14:26:27,221 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/1da6857a570d43d5ae342084aba17d99, entries=150, sequenceid=328, filesize=12.0 K 2024-11-23T14:26:27,221 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/C/a14182a9ee9a490fbde7bc619ddea4fb as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/a14182a9ee9a490fbde7bc619ddea4fb 2024-11-23T14:26:27,227 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/a14182a9ee9a490fbde7bc619ddea4fb, entries=150, sequenceid=328, filesize=12.0 K 2024-11-23T14:26:27,228 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for baf9478a6ce915eeb361816d4eed6489 in 546ms, sequenceid=328, compaction requested=false 2024-11-23T14:26:27,228 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for baf9478a6ce915eeb361816d4eed6489: 2024-11-23T14:26:27,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on baf9478a6ce915eeb361816d4eed6489 2024-11-23T14:26:27,324 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T14:26:27,324 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing baf9478a6ce915eeb361816d4eed6489 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-23T14:26:27,330 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/A/202796d8ff424f32b4c3338e5dfa3a68 is 50, key is test_row_0/A:col10/1732371986686/Put/seqid=0 2024-11-23T14:26:27,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742151_1327 (size=12301) 2024-11-23T14:26:27,335 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/A/202796d8ff424f32b4c3338e5dfa3a68 2024-11-23T14:26:27,345 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/B/a33b21772d5647f8b224f28d6ddefba6 is 50, key is test_row_0/B:col10/1732371986686/Put/seqid=0 2024-11-23T14:26:27,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742152_1328 (size=12301) 2024-11-23T14:26:27,359 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/B/a33b21772d5647f8b224f28d6ddefba6 2024-11-23T14:26:27,363 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-23T14:26:27,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:27,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. as already flushing 2024-11-23T14:26:27,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:27,363 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:27,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:27,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:27,367 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/C/fa98b6ad91d443eaa137a465b682b082 is 50, key is test_row_0/C:col10/1732371986686/Put/seqid=0 2024-11-23T14:26:27,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:27,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51122 deadline: 1732372047365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:27,370 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:27,370 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:27,370 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:27,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742153_1329 (size=12301) 2024-11-23T14:26:27,373 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:27,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1732372047367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:27,373 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/C/fa98b6ad91d443eaa137a465b682b082 2024-11-23T14:26:27,373 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:27,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51208 deadline: 1732372047371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:27,374 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:27,374 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:27,374 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:27,374 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:27,374 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:27,374 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:27,379 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/A/202796d8ff424f32b4c3338e5dfa3a68 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/202796d8ff424f32b4c3338e5dfa3a68 2024-11-23T14:26:27,384 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/202796d8ff424f32b4c3338e5dfa3a68, entries=150, sequenceid=340, filesize=12.0 K 2024-11-23T14:26:27,385 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/B/a33b21772d5647f8b224f28d6ddefba6 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/a33b21772d5647f8b224f28d6ddefba6 2024-11-23T14:26:27,394 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/a33b21772d5647f8b224f28d6ddefba6, entries=150, sequenceid=340, filesize=12.0 K 2024-11-23T14:26:27,396 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/C/fa98b6ad91d443eaa137a465b682b082 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/fa98b6ad91d443eaa137a465b682b082 2024-11-23T14:26:27,399 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/fa98b6ad91d443eaa137a465b682b082, entries=150, sequenceid=340, filesize=12.0 K 2024-11-23T14:26:27,400 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for baf9478a6ce915eeb361816d4eed6489 in 76ms, sequenceid=340, compaction requested=true 2024-11-23T14:26:27,400 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for baf9478a6ce915eeb361816d4eed6489: 2024-11-23T14:26:27,400 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:26:27,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store baf9478a6ce915eeb361816d4eed6489:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T14:26:27,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:26:27,401 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:26:27,402 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:26:27,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store baf9478a6ce915eeb361816d4eed6489:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T14:26:27,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:26:27,402 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1541): baf9478a6ce915eeb361816d4eed6489/B is initiating minor compaction (all files) 2024-11-23T14:26:27,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store baf9478a6ce915eeb361816d4eed6489:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T14:26:27,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:26:27,402 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of baf9478a6ce915eeb361816d4eed6489/B in TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:27,402 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/9310faba50a04f48b4f7e8f36b6b09d7, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/1da6857a570d43d5ae342084aba17d99, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/a33b21772d5647f8b224f28d6ddefba6] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp, totalSize=36.7 K 2024-11-23T14:26:27,402 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40059 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:26:27,402 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1541): baf9478a6ce915eeb361816d4eed6489/A is initiating minor compaction (all files) 2024-11-23T14:26:27,402 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of baf9478a6ce915eeb361816d4eed6489/A in TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:27,402 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/ada6bc81478d4a82b93f8af5469e76af, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/1f71dac9d319448f9072834144085780, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/202796d8ff424f32b4c3338e5dfa3a68] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp, totalSize=39.1 K 2024-11-23T14:26:27,402 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9310faba50a04f48b4f7e8f36b6b09d7, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1732371985491 2024-11-23T14:26:27,403 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting ada6bc81478d4a82b93f8af5469e76af, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1732371985491 2024-11-23T14:26:27,403 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1da6857a570d43d5ae342084aba17d99, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1732371985542 2024-11-23T14:26:27,403 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f71dac9d319448f9072834144085780, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1732371985542 2024-11-23T14:26:27,403 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting a33b21772d5647f8b224f28d6ddefba6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1732371986686 2024-11-23T14:26:27,404 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 202796d8ff424f32b4c3338e5dfa3a68, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1732371986686 2024-11-23T14:26:27,426 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): baf9478a6ce915eeb361816d4eed6489#B#compaction#285 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:26:27,427 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/B/3c9cc3e6cf394ac8acaf235e47ff80ef is 50, key is test_row_0/B:col10/1732371986686/Put/seqid=0 2024-11-23T14:26:27,429 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): baf9478a6ce915eeb361816d4eed6489#A#compaction#286 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:26:27,429 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/A/3626285fd7bf43c0a839ce5057dcb752 is 50, key is test_row_0/A:col10/1732371986686/Put/seqid=0 2024-11-23T14:26:27,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742154_1330 (size=13119) 2024-11-23T14:26:27,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742155_1331 (size=13119) 2024-11-23T14:26:27,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on baf9478a6ce915eeb361816d4eed6489 2024-11-23T14:26:27,483 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T14:26:27,483 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing baf9478a6ce915eeb361816d4eed6489 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-23T14:26:27,489 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/A/c85a1d1b0dde4ee4bb577e1cdb2d1c03 is 50, key is test_row_0/A:col10/1732371987361/Put/seqid=0 2024-11-23T14:26:27,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742156_1332 (size=12301) 2024-11-23T14:26:27,497 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/A/c85a1d1b0dde4ee4bb577e1cdb2d1c03 2024-11-23T14:26:27,499 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:27,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1732372047493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:27,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:27,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:27,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:27,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:27,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51122 deadline: 1732372047498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:27,502 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:27,502 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:27,502 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:27,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:27,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51208 deadline: 1732372047499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:27,503 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:27,503 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:27,503 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:27,509 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/B/fb8a9c91ce9146a6b8c817b79bfe7dd0 is 50, key is test_row_0/B:col10/1732371987361/Put/seqid=0 2024-11-23T14:26:27,516 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-23T14:26:27,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:27,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. as already flushing 2024-11-23T14:26:27,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:27,516 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:27,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:27,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:27,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-23T14:26:27,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742157_1333 (size=12301) 2024-11-23T14:26:27,526 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/B/fb8a9c91ce9146a6b8c817b79bfe7dd0 2024-11-23T14:26:27,534 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/C/e12bdde205ec463484d0ba08f592395d is 50, key is test_row_0/C:col10/1732371987361/Put/seqid=0 2024-11-23T14:26:27,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742158_1334 (size=12301) 2024-11-23T14:26:27,611 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:27,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1732372047610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:27,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:27,612 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:27,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:27,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:27,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51122 deadline: 1732372047610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:27,612 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:27,612 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:27,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51208 deadline: 1732372047610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:27,612 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:27,612 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:27,613 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:27,613 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:27,613 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:27,669 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-23T14:26:27,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:27,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. as already flushing 2024-11-23T14:26:27,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:27,670 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:27,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:27,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:27,825 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:27,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1732372047820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:27,825 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:27,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:27,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:27,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:27,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51208 deadline: 1732372047821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:27,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:27,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51122 deadline: 1732372047821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:27,826 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:27,826 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:27,826 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:27,826 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:27,826 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:27,826 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:27,832 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-23T14:26:27,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:27,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. as already flushing 2024-11-23T14:26:27,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:27,833 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:27,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:27,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:27,854 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/B/3c9cc3e6cf394ac8acaf235e47ff80ef as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/3c9cc3e6cf394ac8acaf235e47ff80ef 2024-11-23T14:26:27,871 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in baf9478a6ce915eeb361816d4eed6489/B of baf9478a6ce915eeb361816d4eed6489 into 3c9cc3e6cf394ac8acaf235e47ff80ef(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:26:27,871 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for baf9478a6ce915eeb361816d4eed6489: 2024-11-23T14:26:27,871 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., storeName=baf9478a6ce915eeb361816d4eed6489/B, priority=13, startTime=1732371987401; duration=0sec 2024-11-23T14:26:27,873 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:26:27,873 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: baf9478a6ce915eeb361816d4eed6489:B 2024-11-23T14:26:27,873 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/A/3626285fd7bf43c0a839ce5057dcb752 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/3626285fd7bf43c0a839ce5057dcb752 2024-11-23T14:26:27,873 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:26:27,875 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:26:27,876 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1541): baf9478a6ce915eeb361816d4eed6489/C is initiating minor compaction (all files) 2024-11-23T14:26:27,876 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of baf9478a6ce915eeb361816d4eed6489/C in TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:27,876 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/74ec07b6e99b469eb30d3692f1867ae1, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/a14182a9ee9a490fbde7bc619ddea4fb, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/fa98b6ad91d443eaa137a465b682b082] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp, totalSize=36.7 K 2024-11-23T14:26:27,876 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 74ec07b6e99b469eb30d3692f1867ae1, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1732371985491 2024-11-23T14:26:27,877 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting a14182a9ee9a490fbde7bc619ddea4fb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1732371985542 2024-11-23T14:26:27,877 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa98b6ad91d443eaa137a465b682b082, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1732371986686 2024-11-23T14:26:27,880 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in baf9478a6ce915eeb361816d4eed6489/A of baf9478a6ce915eeb361816d4eed6489 into 3626285fd7bf43c0a839ce5057dcb752(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:26:27,880 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for baf9478a6ce915eeb361816d4eed6489: 2024-11-23T14:26:27,880 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., storeName=baf9478a6ce915eeb361816d4eed6489/A, priority=13, startTime=1732371987400; duration=0sec 2024-11-23T14:26:27,880 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:26:27,880 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: baf9478a6ce915eeb361816d4eed6489:A 2024-11-23T14:26:27,888 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): baf9478a6ce915eeb361816d4eed6489#C#compaction#290 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:26:27,888 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/C/14a0f0f5194d448c8fc333cc1187c780 is 50, key is test_row_0/C:col10/1732371986686/Put/seqid=0 2024-11-23T14:26:27,949 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/C/e12bdde205ec463484d0ba08f592395d 2024-11-23T14:26:27,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742159_1335 (size=13119) 2024-11-23T14:26:27,957 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/A/c85a1d1b0dde4ee4bb577e1cdb2d1c03 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/c85a1d1b0dde4ee4bb577e1cdb2d1c03 2024-11-23T14:26:27,959 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/C/14a0f0f5194d448c8fc333cc1187c780 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/14a0f0f5194d448c8fc333cc1187c780 2024-11-23T14:26:27,963 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/c85a1d1b0dde4ee4bb577e1cdb2d1c03, entries=150, sequenceid=366, filesize=12.0 K 2024-11-23T14:26:27,964 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/B/fb8a9c91ce9146a6b8c817b79bfe7dd0 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/fb8a9c91ce9146a6b8c817b79bfe7dd0 2024-11-23T14:26:27,966 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in baf9478a6ce915eeb361816d4eed6489/C of baf9478a6ce915eeb361816d4eed6489 into 14a0f0f5194d448c8fc333cc1187c780(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:26:27,966 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for baf9478a6ce915eeb361816d4eed6489: 2024-11-23T14:26:27,966 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., storeName=baf9478a6ce915eeb361816d4eed6489/C, priority=13, startTime=1732371987402; duration=0sec 2024-11-23T14:26:27,966 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:26:27,966 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: baf9478a6ce915eeb361816d4eed6489:C 2024-11-23T14:26:27,969 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/fb8a9c91ce9146a6b8c817b79bfe7dd0, entries=150, sequenceid=366, filesize=12.0 K 2024-11-23T14:26:27,970 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/C/e12bdde205ec463484d0ba08f592395d as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/e12bdde205ec463484d0ba08f592395d 2024-11-23T14:26:27,977 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/e12bdde205ec463484d0ba08f592395d, entries=150, sequenceid=366, filesize=12.0 K 2024-11-23T14:26:27,978 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for baf9478a6ce915eeb361816d4eed6489 in 495ms, sequenceid=366, compaction requested=false 2024-11-23T14:26:27,978 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for baf9478a6ce915eeb361816d4eed6489: 2024-11-23T14:26:27,986 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-23T14:26:27,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:27,986 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing baf9478a6ce915eeb361816d4eed6489 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-23T14:26:27,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/A/4e464a67d9c846fa8036142b5b1725eb is 50, key is test_row_0/A:col10/1732371987497/Put/seqid=0 2024-11-23T14:26:28,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742160_1336 (size=12301) 2024-11-23T14:26:28,011 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/A/4e464a67d9c846fa8036142b5b1725eb 2024-11-23T14:26:28,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/B/ece859a085d540af9ed90ed55f1a9c9c is 50, key is test_row_0/B:col10/1732371987497/Put/seqid=0 2024-11-23T14:26:28,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-23T14:26:28,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742161_1337 (size=12301) 2024-11-23T14:26:28,051 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/B/ece859a085d540af9ed90ed55f1a9c9c 2024-11-23T14:26:28,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/C/1935a357b64a4c9ea0b1b7d97ad96756 is 50, key is test_row_0/C:col10/1732371987497/Put/seqid=0 2024-11-23T14:26:28,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742162_1338 (size=12301) 2024-11-23T14:26:28,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on baf9478a6ce915eeb361816d4eed6489 2024-11-23T14:26:28,166 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. as already flushing 2024-11-23T14:26:28,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:28,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51122 deadline: 1732372048194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:28,198 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:28,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:28,198 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:28,198 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:28,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1732372048195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:28,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:28,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:28,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:28,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:28,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51208 deadline: 1732372048198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:28,203 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:28,203 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:28,203 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:28,302 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:28,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51122 deadline: 1732372048300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:28,303 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:28,303 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:28,303 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:28,312 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:28,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1732372048310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:28,313 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:28,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51208 deadline: 1732372048310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:28,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:28,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:28,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:28,314 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:28,314 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:28,314 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:28,478 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/C/1935a357b64a4c9ea0b1b7d97ad96756 2024-11-23T14:26:28,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/A/4e464a67d9c846fa8036142b5b1725eb as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/4e464a67d9c846fa8036142b5b1725eb 2024-11-23T14:26:28,489 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/4e464a67d9c846fa8036142b5b1725eb, entries=150, sequenceid=379, filesize=12.0 K 2024-11-23T14:26:28,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/B/ece859a085d540af9ed90ed55f1a9c9c as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/ece859a085d540af9ed90ed55f1a9c9c 2024-11-23T14:26:28,496 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/ece859a085d540af9ed90ed55f1a9c9c, entries=150, sequenceid=379, filesize=12.0 K 2024-11-23T14:26:28,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/C/1935a357b64a4c9ea0b1b7d97ad96756 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/1935a357b64a4c9ea0b1b7d97ad96756 2024-11-23T14:26:28,505 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/1935a357b64a4c9ea0b1b7d97ad96756, entries=150, sequenceid=379, filesize=12.0 K 2024-11-23T14:26:28,505 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for baf9478a6ce915eeb361816d4eed6489 in 519ms, sequenceid=379, compaction requested=true 2024-11-23T14:26:28,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for baf9478a6ce915eeb361816d4eed6489: 2024-11-23T14:26:28,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:28,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-23T14:26:28,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-23T14:26:28,513 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T14:26:28,513 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing baf9478a6ce915eeb361816d4eed6489 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-23T14:26:28,514 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-23T14:26:28,514 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6090 sec 2024-11-23T14:26:28,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on baf9478a6ce915eeb361816d4eed6489 2024-11-23T14:26:28,517 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 1.6240 sec 2024-11-23T14:26:28,521 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/A/af3456f8703f42729240bc82b6cfa0c7 is 50, key is test_row_0/A:col10/1732371988185/Put/seqid=0 2024-11-23T14:26:28,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:28,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1732372048535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:28,539 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:28,539 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:28,539 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:28,541 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:28,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51122 deadline: 1732372048538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:28,542 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:28,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51208 deadline: 1732372048538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:28,542 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:28,542 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:28,542 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:28,543 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:28,543 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:28,543 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:28,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742163_1339 (size=14741) 2024-11-23T14:26:28,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:28,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1732372048650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:28,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:28,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:28,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51122 deadline: 1732372048650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:28,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:28,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:28,652 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:28,652 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:28,652 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:28,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:28,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51208 deadline: 1732372048651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:28,652 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:28,652 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:28,652 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:28,861 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:28,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1732372048860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:28,861 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:28,861 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:28,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51122 deadline: 1732372048860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:28,861 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:28,861 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:28,861 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:28,861 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:28,861 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:28,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51208 deadline: 1732372048860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:28,861 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:28,862 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:28,862 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:28,862 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:28,970 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/A/af3456f8703f42729240bc82b6cfa0c7 2024-11-23T14:26:28,981 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/B/8bc30723717548f9878cd551a6a1368f is 50, key is test_row_0/B:col10/1732371988185/Put/seqid=0 2024-11-23T14:26:28,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742164_1340 (size=12301) 2024-11-23T14:26:29,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-23T14:26:29,040 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T14:26:29,040 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-23T14:26:29,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees 2024-11-23T14:26:29,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-23T14:26:29,042 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T14:26:29,042 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T14:26:29,043 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T14:26:29,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-23T14:26:29,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:29,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1732372049170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:29,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:29,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:29,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:29,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:29,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51122 deadline: 1732372049170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:29,173 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:29,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:29,173 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:29,173 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:29,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51208 deadline: 1732372049170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:29,173 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:29,173 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:29,173 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:29,195 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-23T14:26:29,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:29,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. as already flushing 2024-11-23T14:26:29,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:29,195 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:29,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:29,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:29,347 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-23T14:26:29,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:29,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. as already flushing 2024-11-23T14:26:29,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:29,348 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:29,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:29,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:29,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-23T14:26:29,394 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/B/8bc30723717548f9878cd551a6a1368f 2024-11-23T14:26:29,401 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/C/1bc221f960a943bb8866dbdc32c957d8 is 50, key is test_row_0/C:col10/1732371988185/Put/seqid=0 2024-11-23T14:26:29,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742165_1341 (size=12301) 2024-11-23T14:26:29,500 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-23T14:26:29,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:29,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. as already flushing 2024-11-23T14:26:29,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:29,500 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:29,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:29,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:29,653 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-23T14:26:29,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:29,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. as already flushing 2024-11-23T14:26:29,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:29,653 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:29,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:29,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:29,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-23T14:26:29,680 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:29,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1732372049680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:29,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:29,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:29,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:29,682 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:29,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51122 deadline: 1732372049680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:29,683 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:29,683 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:29,683 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:29,683 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:29,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51208 deadline: 1732372049680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:29,683 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:29,683 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:29,683 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:29,805 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/C/1bc221f960a943bb8866dbdc32c957d8 2024-11-23T14:26:29,806 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-23T14:26:29,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:29,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. as already flushing 2024-11-23T14:26:29,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:29,806 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:29,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:29,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:29,810 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/A/af3456f8703f42729240bc82b6cfa0c7 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/af3456f8703f42729240bc82b6cfa0c7 2024-11-23T14:26:29,813 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/af3456f8703f42729240bc82b6cfa0c7, entries=200, sequenceid=406, filesize=14.4 K 2024-11-23T14:26:29,813 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/B/8bc30723717548f9878cd551a6a1368f as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/8bc30723717548f9878cd551a6a1368f 2024-11-23T14:26:29,816 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/8bc30723717548f9878cd551a6a1368f, entries=150, sequenceid=406, filesize=12.0 K 2024-11-23T14:26:29,817 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/C/1bc221f960a943bb8866dbdc32c957d8 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/1bc221f960a943bb8866dbdc32c957d8 2024-11-23T14:26:29,820 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/1bc221f960a943bb8866dbdc32c957d8, entries=150, sequenceid=406, filesize=12.0 K 2024-11-23T14:26:29,821 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=46.96 KB/48090 for baf9478a6ce915eeb361816d4eed6489 in 1308ms, sequenceid=406, compaction requested=true 2024-11-23T14:26:29,821 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for baf9478a6ce915eeb361816d4eed6489: 2024-11-23T14:26:29,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store baf9478a6ce915eeb361816d4eed6489:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T14:26:29,822 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T14:26:29,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:26:29,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store baf9478a6ce915eeb361816d4eed6489:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T14:26:29,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:26:29,822 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T14:26:29,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store baf9478a6ce915eeb361816d4eed6489:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T14:26:29,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:26:29,823 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52462 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T14:26:29,823 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50022 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T14:26:29,823 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1541): baf9478a6ce915eeb361816d4eed6489/A is initiating minor compaction (all files) 2024-11-23T14:26:29,823 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1541): baf9478a6ce915eeb361816d4eed6489/B is initiating minor compaction (all files) 2024-11-23T14:26:29,823 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of baf9478a6ce915eeb361816d4eed6489/A in TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:29,823 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of baf9478a6ce915eeb361816d4eed6489/B in TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:29,823 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/3626285fd7bf43c0a839ce5057dcb752, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/c85a1d1b0dde4ee4bb577e1cdb2d1c03, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/4e464a67d9c846fa8036142b5b1725eb, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/af3456f8703f42729240bc82b6cfa0c7] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp, totalSize=51.2 K 2024-11-23T14:26:29,823 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/3c9cc3e6cf394ac8acaf235e47ff80ef, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/fb8a9c91ce9146a6b8c817b79bfe7dd0, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/ece859a085d540af9ed90ed55f1a9c9c, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/8bc30723717548f9878cd551a6a1368f] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp, totalSize=48.8 K 2024-11-23T14:26:29,824 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c9cc3e6cf394ac8acaf235e47ff80ef, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1732371986686 2024-11-23T14:26:29,824 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3626285fd7bf43c0a839ce5057dcb752, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1732371986686 2024-11-23T14:26:29,824 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting fb8a9c91ce9146a6b8c817b79bfe7dd0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1732371987361 2024-11-23T14:26:29,824 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting c85a1d1b0dde4ee4bb577e1cdb2d1c03, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1732371987361 2024-11-23T14:26:29,824 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e464a67d9c846fa8036142b5b1725eb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1732371987491 2024-11-23T14:26:29,824 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting ece859a085d540af9ed90ed55f1a9c9c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1732371987491 2024-11-23T14:26:29,825 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 8bc30723717548f9878cd551a6a1368f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=406, earliestPutTs=1732371988185 2024-11-23T14:26:29,825 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting af3456f8703f42729240bc82b6cfa0c7, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=406, earliestPutTs=1732371988185 2024-11-23T14:26:29,832 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): baf9478a6ce915eeb361816d4eed6489#B#compaction#297 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:26:29,832 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/B/504f00c35bef495bb510bab08e1a17c0 is 50, key is test_row_0/B:col10/1732371988185/Put/seqid=0 2024-11-23T14:26:29,833 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): baf9478a6ce915eeb361816d4eed6489#A#compaction#298 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:26:29,834 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/A/636dec03bea2428d975fc0fc316a0db1 is 50, key is test_row_0/A:col10/1732371988185/Put/seqid=0 2024-11-23T14:26:29,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742166_1342 (size=13255) 2024-11-23T14:26:29,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742167_1343 (size=13255) 2024-11-23T14:26:29,957 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-23T14:26:29,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:29,958 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2902): Flushing baf9478a6ce915eeb361816d4eed6489 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-23T14:26:29,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/A/94737355e91749d4a7566e9ae4c2480f is 50, key is test_row_0/A:col10/1732371988534/Put/seqid=0 2024-11-23T14:26:29,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742168_1344 (size=12301) 2024-11-23T14:26:30,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on baf9478a6ce915eeb361816d4eed6489 2024-11-23T14:26:30,023 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. as already flushing 2024-11-23T14:26:30,114 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:30,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51138 deadline: 1732372050111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:30,114 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:30,114 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:30,114 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:30,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-23T14:26:30,222 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:30,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51138 deadline: 1732372050220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:30,222 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:30,222 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:30,222 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:30,242 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/B/504f00c35bef495bb510bab08e1a17c0 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/504f00c35bef495bb510bab08e1a17c0 2024-11-23T14:26:30,246 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in baf9478a6ce915eeb361816d4eed6489/B of baf9478a6ce915eeb361816d4eed6489 into 504f00c35bef495bb510bab08e1a17c0(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:26:30,246 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/A/636dec03bea2428d975fc0fc316a0db1 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/636dec03bea2428d975fc0fc316a0db1 2024-11-23T14:26:30,246 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for baf9478a6ce915eeb361816d4eed6489: 2024-11-23T14:26:30,246 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., storeName=baf9478a6ce915eeb361816d4eed6489/B, priority=12, startTime=1732371989822; duration=0sec 2024-11-23T14:26:30,246 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:26:30,246 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: baf9478a6ce915eeb361816d4eed6489:B 2024-11-23T14:26:30,246 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T14:26:30,247 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50022 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T14:26:30,247 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1541): baf9478a6ce915eeb361816d4eed6489/C is initiating minor compaction (all files) 2024-11-23T14:26:30,247 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of baf9478a6ce915eeb361816d4eed6489/C in TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:30,247 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/14a0f0f5194d448c8fc333cc1187c780, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/e12bdde205ec463484d0ba08f592395d, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/1935a357b64a4c9ea0b1b7d97ad96756, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/1bc221f960a943bb8866dbdc32c957d8] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp, totalSize=48.8 K 2024-11-23T14:26:30,248 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 14a0f0f5194d448c8fc333cc1187c780, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1732371986686 2024-11-23T14:26:30,248 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting e12bdde205ec463484d0ba08f592395d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1732371987361 2024-11-23T14:26:30,248 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 1935a357b64a4c9ea0b1b7d97ad96756, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1732371987491 2024-11-23T14:26:30,248 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 1bc221f960a943bb8866dbdc32c957d8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=406, earliestPutTs=1732371988185 2024-11-23T14:26:30,250 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in baf9478a6ce915eeb361816d4eed6489/A of baf9478a6ce915eeb361816d4eed6489 into 636dec03bea2428d975fc0fc316a0db1(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:26:30,250 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for baf9478a6ce915eeb361816d4eed6489: 2024-11-23T14:26:30,250 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., storeName=baf9478a6ce915eeb361816d4eed6489/A, priority=12, startTime=1732371989822; duration=0sec 2024-11-23T14:26:30,250 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:26:30,250 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: baf9478a6ce915eeb361816d4eed6489:A 2024-11-23T14:26:30,256 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): baf9478a6ce915eeb361816d4eed6489#C#compaction#300 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:26:30,256 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/C/c8e230df907545b99a89de1323000233 is 50, key is test_row_0/C:col10/1732371988185/Put/seqid=0 2024-11-23T14:26:30,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742169_1345 (size=13255) 2024-11-23T14:26:30,365 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/A/94737355e91749d4a7566e9ae4c2480f 2024-11-23T14:26:30,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/B/249e8f97222e49d681ef66932b96e6e6 is 50, key is test_row_0/B:col10/1732371988534/Put/seqid=0 2024-11-23T14:26:30,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742170_1346 (size=12301) 2024-11-23T14:26:30,402 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/B/249e8f97222e49d681ef66932b96e6e6 2024-11-23T14:26:30,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/C/dfcb9cc76e1243f6af91300190279aaa is 50, key is test_row_0/C:col10/1732371988534/Put/seqid=0 2024-11-23T14:26:30,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742171_1347 (size=12301) 2024-11-23T14:26:30,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:30,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51138 deadline: 1732372050430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:30,431 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:30,431 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:30,431 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:30,666 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/C/c8e230df907545b99a89de1323000233 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/c8e230df907545b99a89de1323000233 2024-11-23T14:26:30,670 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in baf9478a6ce915eeb361816d4eed6489/C of baf9478a6ce915eeb361816d4eed6489 into c8e230df907545b99a89de1323000233(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:26:30,670 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for baf9478a6ce915eeb361816d4eed6489: 2024-11-23T14:26:30,670 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., storeName=baf9478a6ce915eeb361816d4eed6489/C, priority=12, startTime=1732371989822; duration=0sec 2024-11-23T14:26:30,670 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:26:30,671 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: baf9478a6ce915eeb361816d4eed6489:C 2024-11-23T14:26:30,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:30,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51188 deadline: 1732372050690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:30,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:30,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51122 deadline: 1732372050690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:30,692 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to c6f7b1897123,35385,1732371902767 for 'test_row_2' in baf9478a6ce915eeb361816d4eed6489 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-23T14:26:30,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:30,692 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to c6f7b1897123,35385,1732371902767 for 'test_row_0' in baf9478a6ce915eeb361816d4eed6489 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2154 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-23T14:26:30,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:30,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:30,693 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:30,693 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:30,693 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:30,701 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:30,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51208 deadline: 1732372050699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:30,701 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to c6f7b1897123,35385,1732371902767 for 'test_row_0' in baf9478a6ce915eeb361816d4eed6489 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-23T14:26:30,702 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:30,702 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:30,702 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:30,743 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:30,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51138 deadline: 1732372050740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:30,743 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:30,743 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:30,743 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:30,797 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-23T14:26:30,812 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/C/dfcb9cc76e1243f6af91300190279aaa 2024-11-23T14:26:30,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/A/94737355e91749d4a7566e9ae4c2480f as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/94737355e91749d4a7566e9ae4c2480f 2024-11-23T14:26:30,819 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/94737355e91749d4a7566e9ae4c2480f, entries=150, sequenceid=416, filesize=12.0 K 2024-11-23T14:26:30,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/B/249e8f97222e49d681ef66932b96e6e6 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/249e8f97222e49d681ef66932b96e6e6 2024-11-23T14:26:30,823 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/249e8f97222e49d681ef66932b96e6e6, entries=150, sequenceid=416, filesize=12.0 K 2024-11-23T14:26:30,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/C/dfcb9cc76e1243f6af91300190279aaa as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/dfcb9cc76e1243f6af91300190279aaa 2024-11-23T14:26:30,828 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/dfcb9cc76e1243f6af91300190279aaa, entries=150, sequenceid=416, filesize=12.0 K 2024-11-23T14:26:30,829 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for baf9478a6ce915eeb361816d4eed6489 in 871ms, sequenceid=416, compaction requested=false 2024-11-23T14:26:30,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2603): Flush status journal for baf9478a6ce915eeb361816d4eed6489: 2024-11-23T14:26:30,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:30,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=83 2024-11-23T14:26:30,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster(4169): Remote procedure done, pid=83 2024-11-23T14:26:30,832 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-23T14:26:30,832 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7870 sec 2024-11-23T14:26:30,834 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees in 1.7930 sec 2024-11-23T14:26:31,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-23T14:26:31,190 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T14:26:31,191 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-23T14:26:31,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees 2024-11-23T14:26:31,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-23T14:26:31,192 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T14:26:31,193 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T14:26:31,193 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T14:26:31,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on baf9478a6ce915eeb361816d4eed6489 2024-11-23T14:26:31,252 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T14:26:31,252 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing baf9478a6ce915eeb361816d4eed6489 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-23T14:26:31,257 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/A/af505e9b95ca445c96cb02178604944e is 50, key is test_row_0/A:col10/1732371991251/Put/seqid=0 2024-11-23T14:26:31,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742172_1348 (size=14741) 2024-11-23T14:26:31,274 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:31,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51138 deadline: 1732372051272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:31,274 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:31,275 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:31,275 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:31,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-23T14:26:31,345 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-23T14:26:31,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:31,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. as already flushing 2024-11-23T14:26:31,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:31,345 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:31,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:31,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:31,381 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:31,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51138 deadline: 1732372051380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:31,381 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:31,381 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:31,381 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:31,470 INFO [Thread-1204 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1204. 2024-11-23T14:26:31,470 INFO [Thread-1200 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1200. 2024-11-23T14:26:31,470 DEBUG [Thread-1204 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T14:26:31,470 DEBUG [Thread-1200 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T14:26:31,470 DEBUG [Thread-1204 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:26:31,470 DEBUG [Thread-1200 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:26:31,470 DEBUG [Thread-1200 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:26:31,470 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T14:26:31,470 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T14:26:31,470 DEBUG [Thread-1204 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:26:31,471 INFO [Thread-1202 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1202. 2024-11-23T14:26:31,471 DEBUG [Thread-1202 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T14:26:31,471 DEBUG [Thread-1202 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:26:31,471 DEBUG [Thread-1202 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:26:31,471 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T14:26:31,472 INFO [Thread-1208 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1208. 2024-11-23T14:26:31,473 DEBUG [Thread-1208 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T14:26:31,473 DEBUG [Thread-1208 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:26:31,473 DEBUG [Thread-1208 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:26:31,473 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T14:26:31,473 INFO [Thread-1206 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1206. 2024-11-23T14:26:31,473 DEBUG [Thread-1206 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T14:26:31,473 DEBUG [Thread-1206 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:26:31,473 DEBUG [Thread-1206 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:26:31,473 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T14:26:31,498 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-23T14:26:31,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:31,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. as already flushing 2024-11-23T14:26:31,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:31,498 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:31,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:31,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:31,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-23T14:26:31,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:31,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51138 deadline: 1732372051591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:31,593 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:31,593 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:31,593 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:31,652 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-23T14:26:31,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:31,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. as already flushing 2024-11-23T14:26:31,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:31,653 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:31,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:31,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:31,662 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/A/af505e9b95ca445c96cb02178604944e 2024-11-23T14:26:31,669 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/B/36621f9ce50b4824aadc1ce050e37722 is 50, key is test_row_0/B:col10/1732371991251/Put/seqid=0 2024-11-23T14:26:31,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742173_1349 (size=12301) 2024-11-23T14:26:31,810 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-23T14:26:31,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:31,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. as already flushing 2024-11-23T14:26:31,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:31,811 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:31,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:31,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:31,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-23T14:26:31,902 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:31,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51138 deadline: 1732372051901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:31,903 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:31,903 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:31,903 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:31,969 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-23T14:26:31,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:31,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. as already flushing 2024-11-23T14:26:31,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:31,970 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:31,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:31,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:32,073 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/B/36621f9ce50b4824aadc1ce050e37722 2024-11-23T14:26:32,085 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/C/2c7ef7a7567a4d55aef6e7a0ecba26d5 is 50, key is test_row_0/C:col10/1732371991251/Put/seqid=0 2024-11-23T14:26:32,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742174_1350 (size=12301) 2024-11-23T14:26:32,125 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-23T14:26:32,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:32,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. as already flushing 2024-11-23T14:26:32,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:32,126 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:32,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:32,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:32,279 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-23T14:26:32,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:32,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. as already flushing 2024-11-23T14:26:32,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:32,280 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:32,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:32,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:32,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-23T14:26:32,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:32,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51138 deadline: 1732372052411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:32,413 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:32,413 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:32,413 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:26:32,434 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-23T14:26:32,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:32,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. as already flushing 2024-11-23T14:26:32,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:32,435 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:32,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:32,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:32,490 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/C/2c7ef7a7567a4d55aef6e7a0ecba26d5 2024-11-23T14:26:32,501 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/A/af505e9b95ca445c96cb02178604944e as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/af505e9b95ca445c96cb02178604944e 2024-11-23T14:26:32,506 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/af505e9b95ca445c96cb02178604944e, entries=200, sequenceid=446, filesize=14.4 K 2024-11-23T14:26:32,507 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/B/36621f9ce50b4824aadc1ce050e37722 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/36621f9ce50b4824aadc1ce050e37722 2024-11-23T14:26:32,510 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/36621f9ce50b4824aadc1ce050e37722, entries=150, sequenceid=446, filesize=12.0 K 2024-11-23T14:26:32,511 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/C/2c7ef7a7567a4d55aef6e7a0ecba26d5 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/2c7ef7a7567a4d55aef6e7a0ecba26d5 2024-11-23T14:26:32,514 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/2c7ef7a7567a4d55aef6e7a0ecba26d5, entries=150, sequenceid=446, filesize=12.0 K 2024-11-23T14:26:32,515 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for baf9478a6ce915eeb361816d4eed6489 in 1263ms, sequenceid=446, compaction requested=true 2024-11-23T14:26:32,515 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for baf9478a6ce915eeb361816d4eed6489: 2024-11-23T14:26:32,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store baf9478a6ce915eeb361816d4eed6489:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T14:26:32,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:26:32,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store baf9478a6ce915eeb361816d4eed6489:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T14:26:32,516 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:26:32,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:26:32,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store baf9478a6ce915eeb361816d4eed6489:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T14:26:32,516 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:26:32,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:26:32,516 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:26:32,516 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40297 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:26:32,516 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1541): baf9478a6ce915eeb361816d4eed6489/B is initiating minor compaction (all files) 2024-11-23T14:26:32,516 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1541): baf9478a6ce915eeb361816d4eed6489/A is initiating minor compaction (all files) 2024-11-23T14:26:32,516 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of baf9478a6ce915eeb361816d4eed6489/B in TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:32,516 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of baf9478a6ce915eeb361816d4eed6489/A in TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:32,517 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/504f00c35bef495bb510bab08e1a17c0, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/249e8f97222e49d681ef66932b96e6e6, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/36621f9ce50b4824aadc1ce050e37722] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp, totalSize=37.0 K 2024-11-23T14:26:32,517 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/636dec03bea2428d975fc0fc316a0db1, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/94737355e91749d4a7566e9ae4c2480f, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/af505e9b95ca445c96cb02178604944e] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp, totalSize=39.4 K 2024-11-23T14:26:32,517 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 504f00c35bef495bb510bab08e1a17c0, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=406, earliestPutTs=1732371988185 2024-11-23T14:26:32,517 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 636dec03bea2428d975fc0fc316a0db1, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=406, earliestPutTs=1732371988185 2024-11-23T14:26:32,517 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 249e8f97222e49d681ef66932b96e6e6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1732371988524 2024-11-23T14:26:32,517 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 94737355e91749d4a7566e9ae4c2480f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1732371988524 2024-11-23T14:26:32,517 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 36621f9ce50b4824aadc1ce050e37722, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=446, earliestPutTs=1732371990086 2024-11-23T14:26:32,517 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting af505e9b95ca445c96cb02178604944e, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=446, earliestPutTs=1732371990086 2024-11-23T14:26:32,523 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): baf9478a6ce915eeb361816d4eed6489#A#compaction#306 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:26:32,523 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/A/58ca4f4799c54b79b0d3c881b795d659 is 50, key is test_row_0/A:col10/1732371991251/Put/seqid=0 2024-11-23T14:26:32,524 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): baf9478a6ce915eeb361816d4eed6489#B#compaction#307 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:26:32,524 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/B/ca9cd2e8457a4a978cd33fdf03a2e790 is 50, key is test_row_0/B:col10/1732371991251/Put/seqid=0 2024-11-23T14:26:32,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742175_1351 (size=13357) 2024-11-23T14:26:32,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742176_1352 (size=13357) 2024-11-23T14:26:32,589 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-23T14:26:32,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:32,589 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2902): Flushing baf9478a6ce915eeb361816d4eed6489 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-23T14:26:32,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/A/936cc91260ed4240a4ebe080edc81dfa is 50, key is test_row_0/A:col10/1732371991260/Put/seqid=0 2024-11-23T14:26:32,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742177_1353 (size=12301) 2024-11-23T14:26:32,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on baf9478a6ce915eeb361816d4eed6489 2024-11-23T14:26:32,716 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. as already flushing 2024-11-23T14:26:32,716 INFO [Thread-1191 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1191. 2024-11-23T14:26:32,716 DEBUG [Thread-1191 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T14:26:32,716 DEBUG [Thread-1191 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:26:32,716 INFO [Thread-1193 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1193. 2024-11-23T14:26:32,716 DEBUG [Thread-1193 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T14:26:32,716 DEBUG [Thread-1191 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:26:32,717 DEBUG [Thread-1193 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:26:32,717 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T14:26:32,717 DEBUG [Thread-1193 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:26:32,717 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T14:26:32,722 INFO [Thread-1197 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1197. 2024-11-23T14:26:32,722 DEBUG [Thread-1197 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T14:26:32,722 DEBUG [Thread-1197 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:26:32,723 DEBUG [Thread-1197 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:26:32,723 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T14:26:32,932 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/A/58ca4f4799c54b79b0d3c881b795d659 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/58ca4f4799c54b79b0d3c881b795d659 2024-11-23T14:26:32,934 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/B/ca9cd2e8457a4a978cd33fdf03a2e790 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/ca9cd2e8457a4a978cd33fdf03a2e790 2024-11-23T14:26:32,939 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in baf9478a6ce915eeb361816d4eed6489/A of baf9478a6ce915eeb361816d4eed6489 into 58ca4f4799c54b79b0d3c881b795d659(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:26:32,939 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for baf9478a6ce915eeb361816d4eed6489: 2024-11-23T14:26:32,939 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., storeName=baf9478a6ce915eeb361816d4eed6489/A, priority=13, startTime=1732371992515; duration=0sec 2024-11-23T14:26:32,940 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:26:32,940 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: baf9478a6ce915eeb361816d4eed6489:A 2024-11-23T14:26:32,940 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:26:32,941 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:26:32,941 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1541): baf9478a6ce915eeb361816d4eed6489/C is initiating minor compaction (all files) 2024-11-23T14:26:32,941 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in baf9478a6ce915eeb361816d4eed6489/B of baf9478a6ce915eeb361816d4eed6489 into ca9cd2e8457a4a978cd33fdf03a2e790(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:26:32,941 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of baf9478a6ce915eeb361816d4eed6489/C in TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:32,941 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for baf9478a6ce915eeb361816d4eed6489: 2024-11-23T14:26:32,941 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., storeName=baf9478a6ce915eeb361816d4eed6489/B, priority=13, startTime=1732371992516; duration=0sec 2024-11-23T14:26:32,941 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/c8e230df907545b99a89de1323000233, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/dfcb9cc76e1243f6af91300190279aaa, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/2c7ef7a7567a4d55aef6e7a0ecba26d5] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp, totalSize=37.0 K 2024-11-23T14:26:32,941 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:26:32,941 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: baf9478a6ce915eeb361816d4eed6489:B 2024-11-23T14:26:32,941 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting c8e230df907545b99a89de1323000233, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=406, earliestPutTs=1732371988185 2024-11-23T14:26:32,941 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting dfcb9cc76e1243f6af91300190279aaa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1732371988524 2024-11-23T14:26:32,942 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2c7ef7a7567a4d55aef6e7a0ecba26d5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=446, earliestPutTs=1732371990086 2024-11-23T14:26:32,947 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): baf9478a6ce915eeb361816d4eed6489#C#compaction#309 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:26:32,948 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/C/be27de1eb7304afbbb03d74012c45f3d is 50, key is test_row_0/C:col10/1732371991251/Put/seqid=0 2024-11-23T14:26:32,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742178_1354 (size=13357) 2024-11-23T14:26:33,000 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=455 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/A/936cc91260ed4240a4ebe080edc81dfa 2024-11-23T14:26:33,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/B/cd63bbd3c0ef4a7baddb8ebb108b507c is 50, key is test_row_0/B:col10/1732371991260/Put/seqid=0 2024-11-23T14:26:33,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742179_1355 (size=12301) 2024-11-23T14:26:33,266 INFO [Thread-1195 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1195. 2024-11-23T14:26:33,266 DEBUG [Thread-1195 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T14:26:33,266 DEBUG [Thread-1195 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:26:33,266 DEBUG [Thread-1195 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:26:33,266 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T14:26:33,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-23T14:26:33,362 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/C/be27de1eb7304afbbb03d74012c45f3d as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/be27de1eb7304afbbb03d74012c45f3d 2024-11-23T14:26:33,366 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in baf9478a6ce915eeb361816d4eed6489/C of baf9478a6ce915eeb361816d4eed6489 into be27de1eb7304afbbb03d74012c45f3d(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:26:33,366 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for baf9478a6ce915eeb361816d4eed6489: 2024-11-23T14:26:33,367 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., storeName=baf9478a6ce915eeb361816d4eed6489/C, priority=13, startTime=1732371992516; duration=0sec 2024-11-23T14:26:33,367 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:26:33,367 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: baf9478a6ce915eeb361816d4eed6489:C 2024-11-23T14:26:33,410 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=455 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/B/cd63bbd3c0ef4a7baddb8ebb108b507c 2024-11-23T14:26:33,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/C/ee58f9163f5b48ef88899ddf4a94bd05 is 50, key is test_row_0/C:col10/1732371991260/Put/seqid=0 2024-11-23T14:26:33,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742180_1356 (size=12301) 2024-11-23T14:26:33,432 INFO [Thread-1189 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1189. 2024-11-23T14:26:33,432 DEBUG [Thread-1189 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T14:26:33,432 DEBUG [Thread-1189 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:26:33,432 DEBUG [Thread-1189 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:26:33,432 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T14:26:33,825 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=455 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/C/ee58f9163f5b48ef88899ddf4a94bd05 2024-11-23T14:26:33,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/A/936cc91260ed4240a4ebe080edc81dfa as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/936cc91260ed4240a4ebe080edc81dfa 2024-11-23T14:26:33,833 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/936cc91260ed4240a4ebe080edc81dfa, entries=150, sequenceid=455, filesize=12.0 K 2024-11-23T14:26:33,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/B/cd63bbd3c0ef4a7baddb8ebb108b507c as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/cd63bbd3c0ef4a7baddb8ebb108b507c 2024-11-23T14:26:33,837 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/cd63bbd3c0ef4a7baddb8ebb108b507c, entries=150, sequenceid=455, filesize=12.0 K 2024-11-23T14:26:33,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/C/ee58f9163f5b48ef88899ddf4a94bd05 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/ee58f9163f5b48ef88899ddf4a94bd05 2024-11-23T14:26:33,841 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/ee58f9163f5b48ef88899ddf4a94bd05, entries=150, sequenceid=455, filesize=12.0 K 2024-11-23T14:26:33,842 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=33.54 KB/34350 for baf9478a6ce915eeb361816d4eed6489 in 1253ms, sequenceid=455, compaction requested=false 2024-11-23T14:26:33,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2603): Flush status journal for baf9478a6ce915eeb361816d4eed6489: 2024-11-23T14:26:33,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:33,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=85 2024-11-23T14:26:33,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster(4169): Remote procedure done, pid=85 2024-11-23T14:26:33,845 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-23T14:26:33,845 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6500 sec 2024-11-23T14:26:33,846 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees in 2.6540 sec 2024-11-23T14:26:35,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-23T14:26:35,351 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T14:26:35,351 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-23T14:26:35,351 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 142 2024-11-23T14:26:35,351 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 68 2024-11-23T14:26:35,351 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 63 2024-11-23T14:26:35,351 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 23 2024-11-23T14:26:35,351 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 67 2024-11-23T14:26:35,351 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-23T14:26:35,351 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3871 2024-11-23T14:26:35,351 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3808 2024-11-23T14:26:35,351 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3739 2024-11-23T14:26:35,351 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3865 2024-11-23T14:26:35,351 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3826 2024-11-23T14:26:35,351 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-23T14:26:35,351 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-23T14:26:35,352 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T14:26:35,352 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:26:35,352 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:26:35,352 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T14:26:35,353 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-23T14:26:35,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-23T14:26:35,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-23T14:26:35,359 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732371995359"}]},"ts":"1732371995359"} 2024-11-23T14:26:35,361 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-23T14:26:35,361 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-23T14:26:35,362 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-23T14:26:35,363 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=baf9478a6ce915eeb361816d4eed6489, UNASSIGN}] 2024-11-23T14:26:35,364 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=baf9478a6ce915eeb361816d4eed6489, UNASSIGN 2024-11-23T14:26:35,364 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=baf9478a6ce915eeb361816d4eed6489, regionState=CLOSING, regionLocation=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:35,365 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=baf9478a6ce915eeb361816d4eed6489, UNASSIGN because future has completed 2024-11-23T14:26:35,366 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-23T14:26:35,366 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE, hasLock=false; CloseRegionProcedure baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767}] 2024-11-23T14:26:35,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-23T14:26:35,518 INFO [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(122): Close baf9478a6ce915eeb361816d4eed6489 2024-11-23T14:26:35,518 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-23T14:26:35,518 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1722): Closing baf9478a6ce915eeb361816d4eed6489, disabling compactions & flushes 2024-11-23T14:26:35,518 INFO [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:35,518 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:35,518 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. after waiting 0 ms 2024-11-23T14:26:35,518 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:35,518 INFO [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(2902): Flushing baf9478a6ce915eeb361816d4eed6489 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-23T14:26:35,522 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/A/f53b6589353c4d29b9823f552349a090 is 50, key is test_row_0/A:col10/1732371992721/Put/seqid=0 2024-11-23T14:26:35,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742181_1357 (size=12301) 2024-11-23T14:26:35,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-23T14:26:35,926 INFO [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=466 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/A/f53b6589353c4d29b9823f552349a090 2024-11-23T14:26:35,934 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/B/c711f89b1b5c42c2a9a0adb2e7bf5cac is 50, key is test_row_0/B:col10/1732371992721/Put/seqid=0 2024-11-23T14:26:35,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742182_1358 (size=12301) 2024-11-23T14:26:35,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-23T14:26:36,339 INFO [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=466 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/B/c711f89b1b5c42c2a9a0adb2e7bf5cac 2024-11-23T14:26:36,353 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/C/994402d56065441abf1cd1eceacfb935 is 50, key is test_row_0/C:col10/1732371992721/Put/seqid=0 2024-11-23T14:26:36,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742183_1359 (size=12301) 2024-11-23T14:26:36,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-23T14:26:36,758 INFO [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=466 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/C/994402d56065441abf1cd1eceacfb935 2024-11-23T14:26:36,769 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/A/f53b6589353c4d29b9823f552349a090 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/f53b6589353c4d29b9823f552349a090 2024-11-23T14:26:36,773 INFO [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/f53b6589353c4d29b9823f552349a090, entries=150, sequenceid=466, filesize=12.0 K 2024-11-23T14:26:36,774 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/B/c711f89b1b5c42c2a9a0adb2e7bf5cac as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/c711f89b1b5c42c2a9a0adb2e7bf5cac 2024-11-23T14:26:36,778 INFO [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/c711f89b1b5c42c2a9a0adb2e7bf5cac, entries=150, sequenceid=466, filesize=12.0 K 2024-11-23T14:26:36,779 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/.tmp/C/994402d56065441abf1cd1eceacfb935 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/994402d56065441abf1cd1eceacfb935 2024-11-23T14:26:36,783 INFO [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/994402d56065441abf1cd1eceacfb935, entries=150, sequenceid=466, filesize=12.0 K 2024-11-23T14:26:36,784 INFO [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for baf9478a6ce915eeb361816d4eed6489 in 1266ms, sequenceid=466, compaction requested=true 2024-11-23T14:26:36,785 DEBUG [StoreCloser-TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/f1fcf5754c4244cf9c2af1f06400b2ef, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/4d630290d9d74b9388b6a0497e61488f, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/bb92a85bfaa74c1facd1d6cb6ef76d27, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/342a3cd26c824a659c805c081b9aa1ef, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/dadacaeec57a4a73b634018f1bc5d500, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/4a8599eac41144e7807ba23235bde840, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/397999acefda4662a1fc269d06d2369f, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/e700eca174de47799af9363037b2cb4c, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/ce872b59e4294deaa509506cea8563b4, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/70b7ac152ca747e38b6c96d06430aa9e, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/a58cda7a68aa49b3ac2147367457446e, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/e42b0246f05f46d09787d379b0ebb886, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/32a3c3998e454e8dab565ea5ebe9488a, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/bd558a1438c1406f860382db8a9da04d, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/dce7a048321c4a7a856c5f37d076a929, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/f6ea0e5866d646f78355422ba85cd217, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/fd1cc12f013540d590b0679f37f345e4, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/07de77a219ab4cc98fab9491ab4608e9, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/2c25ab3e7d364e1182681d2354fda98b, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/72dfd8ea3cd2444bb73d8e57f4eebd50, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/fceec668b5d847e3868876a38cab7f3b, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/ada6bc81478d4a82b93f8af5469e76af, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/1f71dac9d319448f9072834144085780, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/3626285fd7bf43c0a839ce5057dcb752, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/202796d8ff424f32b4c3338e5dfa3a68, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/c85a1d1b0dde4ee4bb577e1cdb2d1c03, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/4e464a67d9c846fa8036142b5b1725eb, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/af3456f8703f42729240bc82b6cfa0c7, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/636dec03bea2428d975fc0fc316a0db1, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/94737355e91749d4a7566e9ae4c2480f, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/af505e9b95ca445c96cb02178604944e] to archive 2024-11-23T14:26:36,786 DEBUG [StoreCloser-TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-23T14:26:36,788 DEBUG [StoreCloser-TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/f1fcf5754c4244cf9c2af1f06400b2ef to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/f1fcf5754c4244cf9c2af1f06400b2ef 2024-11-23T14:26:36,790 DEBUG [StoreCloser-TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/4d630290d9d74b9388b6a0497e61488f to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/4d630290d9d74b9388b6a0497e61488f 2024-11-23T14:26:36,791 DEBUG [StoreCloser-TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/bb92a85bfaa74c1facd1d6cb6ef76d27 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/bb92a85bfaa74c1facd1d6cb6ef76d27 2024-11-23T14:26:36,792 DEBUG [StoreCloser-TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/342a3cd26c824a659c805c081b9aa1ef to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/342a3cd26c824a659c805c081b9aa1ef 2024-11-23T14:26:36,793 DEBUG [StoreCloser-TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/dadacaeec57a4a73b634018f1bc5d500 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/dadacaeec57a4a73b634018f1bc5d500 2024-11-23T14:26:36,794 DEBUG [StoreCloser-TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/4a8599eac41144e7807ba23235bde840 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/4a8599eac41144e7807ba23235bde840 2024-11-23T14:26:36,795 DEBUG [StoreCloser-TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/397999acefda4662a1fc269d06d2369f to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/397999acefda4662a1fc269d06d2369f 2024-11-23T14:26:36,796 DEBUG [StoreCloser-TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/e700eca174de47799af9363037b2cb4c to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/e700eca174de47799af9363037b2cb4c 2024-11-23T14:26:36,797 DEBUG [StoreCloser-TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/ce872b59e4294deaa509506cea8563b4 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/ce872b59e4294deaa509506cea8563b4 2024-11-23T14:26:36,798 DEBUG [StoreCloser-TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/70b7ac152ca747e38b6c96d06430aa9e to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/70b7ac152ca747e38b6c96d06430aa9e 2024-11-23T14:26:36,798 DEBUG [StoreCloser-TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/a58cda7a68aa49b3ac2147367457446e to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/a58cda7a68aa49b3ac2147367457446e 2024-11-23T14:26:36,799 DEBUG [StoreCloser-TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/e42b0246f05f46d09787d379b0ebb886 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/e42b0246f05f46d09787d379b0ebb886 2024-11-23T14:26:36,800 DEBUG [StoreCloser-TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/32a3c3998e454e8dab565ea5ebe9488a to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/32a3c3998e454e8dab565ea5ebe9488a 2024-11-23T14:26:36,801 DEBUG [StoreCloser-TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/bd558a1438c1406f860382db8a9da04d to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/bd558a1438c1406f860382db8a9da04d 2024-11-23T14:26:36,802 DEBUG [StoreCloser-TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/dce7a048321c4a7a856c5f37d076a929 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/dce7a048321c4a7a856c5f37d076a929 2024-11-23T14:26:36,802 DEBUG [StoreCloser-TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/f6ea0e5866d646f78355422ba85cd217 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/f6ea0e5866d646f78355422ba85cd217 2024-11-23T14:26:36,803 DEBUG [StoreCloser-TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/fd1cc12f013540d590b0679f37f345e4 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/fd1cc12f013540d590b0679f37f345e4 2024-11-23T14:26:36,804 DEBUG [StoreCloser-TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/07de77a219ab4cc98fab9491ab4608e9 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/07de77a219ab4cc98fab9491ab4608e9 2024-11-23T14:26:36,805 DEBUG [StoreCloser-TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/2c25ab3e7d364e1182681d2354fda98b to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/2c25ab3e7d364e1182681d2354fda98b 2024-11-23T14:26:36,805 DEBUG [StoreCloser-TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/72dfd8ea3cd2444bb73d8e57f4eebd50 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/72dfd8ea3cd2444bb73d8e57f4eebd50 2024-11-23T14:26:36,806 DEBUG [StoreCloser-TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/fceec668b5d847e3868876a38cab7f3b to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/fceec668b5d847e3868876a38cab7f3b 2024-11-23T14:26:36,807 DEBUG [StoreCloser-TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/ada6bc81478d4a82b93f8af5469e76af to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/ada6bc81478d4a82b93f8af5469e76af 2024-11-23T14:26:36,808 DEBUG [StoreCloser-TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/1f71dac9d319448f9072834144085780 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/1f71dac9d319448f9072834144085780 2024-11-23T14:26:36,808 DEBUG [StoreCloser-TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/3626285fd7bf43c0a839ce5057dcb752 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/3626285fd7bf43c0a839ce5057dcb752 2024-11-23T14:26:36,809 DEBUG [StoreCloser-TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/202796d8ff424f32b4c3338e5dfa3a68 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/202796d8ff424f32b4c3338e5dfa3a68 2024-11-23T14:26:36,810 DEBUG [StoreCloser-TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/c85a1d1b0dde4ee4bb577e1cdb2d1c03 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/c85a1d1b0dde4ee4bb577e1cdb2d1c03 2024-11-23T14:26:36,810 DEBUG [StoreCloser-TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/4e464a67d9c846fa8036142b5b1725eb to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/4e464a67d9c846fa8036142b5b1725eb 2024-11-23T14:26:36,811 DEBUG [StoreCloser-TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/af3456f8703f42729240bc82b6cfa0c7 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/af3456f8703f42729240bc82b6cfa0c7 2024-11-23T14:26:36,811 DEBUG [StoreCloser-TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/636dec03bea2428d975fc0fc316a0db1 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/636dec03bea2428d975fc0fc316a0db1 2024-11-23T14:26:36,812 DEBUG [StoreCloser-TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/94737355e91749d4a7566e9ae4c2480f to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/94737355e91749d4a7566e9ae4c2480f 2024-11-23T14:26:36,813 DEBUG [StoreCloser-TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/af505e9b95ca445c96cb02178604944e to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/af505e9b95ca445c96cb02178604944e 2024-11-23T14:26:36,814 DEBUG [StoreCloser-TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/94006de13e314def8c432f66c67f40ed, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/f67eb7a261474af2b142e77e36d3d431, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/840ebc235b884edc875e75394c7b9336, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/796c4c600e71424f92aa360b64276fe3, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/dd5847abb25f4b5589ebd148d3d547ea, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/74c19b0b15534370a632263b234a6632, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/07bf96bc69e94099a15b52abf2258176, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/39736117dee5489cbe4147cad9cf31ea, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/cb96316e344c443497f62ea7439a84e2, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/536897a9c5ea4f0996e79729ceffa724, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/86e4060b7deb494cbc609c57569e90d7, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/8c94decc660549f9a53e5001f3e1a54b, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/2e74f5a21cf64fba9d211041023c7fde, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/649f81df464e41d5a98e3947a37b89ba, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/bf26281ba7804e9b87af66ee909a0f61, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/2c1968ed8a8c4b1faabaeea718411f04, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/2bb48d0b07944a3db8f50ac704bd91b1, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/8557925f7f3a4204b4d48da94cd683f1, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/c4f57c4c586a4b97b7de50b1c36a96ec, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/f95f96fc0cb34b4ea36e9beb96cf8c69, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/9310faba50a04f48b4f7e8f36b6b09d7, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/a78e270c3de541dd8b0b62208ba8538d, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/1da6857a570d43d5ae342084aba17d99, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/3c9cc3e6cf394ac8acaf235e47ff80ef, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/a33b21772d5647f8b224f28d6ddefba6, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/fb8a9c91ce9146a6b8c817b79bfe7dd0, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/ece859a085d540af9ed90ed55f1a9c9c, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/504f00c35bef495bb510bab08e1a17c0, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/8bc30723717548f9878cd551a6a1368f, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/249e8f97222e49d681ef66932b96e6e6, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/36621f9ce50b4824aadc1ce050e37722] to archive 2024-11-23T14:26:36,815 DEBUG [StoreCloser-TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-23T14:26:36,816 DEBUG [StoreCloser-TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/94006de13e314def8c432f66c67f40ed to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/94006de13e314def8c432f66c67f40ed 2024-11-23T14:26:36,817 DEBUG [StoreCloser-TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/f67eb7a261474af2b142e77e36d3d431 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/f67eb7a261474af2b142e77e36d3d431 2024-11-23T14:26:36,818 DEBUG [StoreCloser-TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/840ebc235b884edc875e75394c7b9336 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/840ebc235b884edc875e75394c7b9336 2024-11-23T14:26:36,819 DEBUG [StoreCloser-TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/796c4c600e71424f92aa360b64276fe3 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/796c4c600e71424f92aa360b64276fe3 2024-11-23T14:26:36,820 DEBUG [StoreCloser-TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/dd5847abb25f4b5589ebd148d3d547ea to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/dd5847abb25f4b5589ebd148d3d547ea 2024-11-23T14:26:36,821 DEBUG [StoreCloser-TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/74c19b0b15534370a632263b234a6632 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/74c19b0b15534370a632263b234a6632 2024-11-23T14:26:36,822 DEBUG [StoreCloser-TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/07bf96bc69e94099a15b52abf2258176 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/07bf96bc69e94099a15b52abf2258176 2024-11-23T14:26:36,823 DEBUG [StoreCloser-TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/39736117dee5489cbe4147cad9cf31ea to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/39736117dee5489cbe4147cad9cf31ea 2024-11-23T14:26:36,824 DEBUG [StoreCloser-TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/cb96316e344c443497f62ea7439a84e2 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/cb96316e344c443497f62ea7439a84e2 2024-11-23T14:26:36,825 DEBUG [StoreCloser-TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/536897a9c5ea4f0996e79729ceffa724 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/536897a9c5ea4f0996e79729ceffa724 2024-11-23T14:26:36,826 DEBUG [StoreCloser-TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/86e4060b7deb494cbc609c57569e90d7 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/86e4060b7deb494cbc609c57569e90d7 2024-11-23T14:26:36,827 DEBUG [StoreCloser-TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/8c94decc660549f9a53e5001f3e1a54b to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/8c94decc660549f9a53e5001f3e1a54b 2024-11-23T14:26:36,828 DEBUG [StoreCloser-TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/2e74f5a21cf64fba9d211041023c7fde to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/2e74f5a21cf64fba9d211041023c7fde 2024-11-23T14:26:36,829 DEBUG [StoreCloser-TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/649f81df464e41d5a98e3947a37b89ba to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/649f81df464e41d5a98e3947a37b89ba 2024-11-23T14:26:36,830 DEBUG [StoreCloser-TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/bf26281ba7804e9b87af66ee909a0f61 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/bf26281ba7804e9b87af66ee909a0f61 2024-11-23T14:26:36,832 DEBUG [StoreCloser-TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/2c1968ed8a8c4b1faabaeea718411f04 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/2c1968ed8a8c4b1faabaeea718411f04 2024-11-23T14:26:36,833 DEBUG [StoreCloser-TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/2bb48d0b07944a3db8f50ac704bd91b1 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/2bb48d0b07944a3db8f50ac704bd91b1 2024-11-23T14:26:36,834 DEBUG [StoreCloser-TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/8557925f7f3a4204b4d48da94cd683f1 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/8557925f7f3a4204b4d48da94cd683f1 2024-11-23T14:26:36,835 DEBUG [StoreCloser-TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/c4f57c4c586a4b97b7de50b1c36a96ec to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/c4f57c4c586a4b97b7de50b1c36a96ec 2024-11-23T14:26:36,836 DEBUG [StoreCloser-TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/f95f96fc0cb34b4ea36e9beb96cf8c69 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/f95f96fc0cb34b4ea36e9beb96cf8c69 2024-11-23T14:26:36,837 DEBUG [StoreCloser-TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/9310faba50a04f48b4f7e8f36b6b09d7 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/9310faba50a04f48b4f7e8f36b6b09d7 2024-11-23T14:26:36,838 DEBUG [StoreCloser-TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/a78e270c3de541dd8b0b62208ba8538d to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/a78e270c3de541dd8b0b62208ba8538d 2024-11-23T14:26:36,839 DEBUG [StoreCloser-TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/1da6857a570d43d5ae342084aba17d99 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/1da6857a570d43d5ae342084aba17d99 2024-11-23T14:26:36,839 DEBUG [StoreCloser-TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/3c9cc3e6cf394ac8acaf235e47ff80ef to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/3c9cc3e6cf394ac8acaf235e47ff80ef 2024-11-23T14:26:36,840 DEBUG [StoreCloser-TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/a33b21772d5647f8b224f28d6ddefba6 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/a33b21772d5647f8b224f28d6ddefba6 2024-11-23T14:26:36,841 DEBUG [StoreCloser-TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/fb8a9c91ce9146a6b8c817b79bfe7dd0 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/fb8a9c91ce9146a6b8c817b79bfe7dd0 2024-11-23T14:26:36,842 DEBUG [StoreCloser-TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/ece859a085d540af9ed90ed55f1a9c9c to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/ece859a085d540af9ed90ed55f1a9c9c 2024-11-23T14:26:36,843 DEBUG [StoreCloser-TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/504f00c35bef495bb510bab08e1a17c0 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/504f00c35bef495bb510bab08e1a17c0 2024-11-23T14:26:36,843 DEBUG [StoreCloser-TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/8bc30723717548f9878cd551a6a1368f to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/8bc30723717548f9878cd551a6a1368f 2024-11-23T14:26:36,844 DEBUG [StoreCloser-TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/249e8f97222e49d681ef66932b96e6e6 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/249e8f97222e49d681ef66932b96e6e6 2024-11-23T14:26:36,845 DEBUG [StoreCloser-TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/36621f9ce50b4824aadc1ce050e37722 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/36621f9ce50b4824aadc1ce050e37722 2024-11-23T14:26:36,846 DEBUG [StoreCloser-TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/37dd9dbf45fd494babf5235cce649607, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/0a7b288bb2284b32999cc74acc111b66, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/3382d99ca2d64d7c8945a5a4d7d6adbf, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/367417322d7747e283ddfbd4c74d2e3d, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/7ca348205a0b4c2cb4173c49f2ce76f4, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/be29c0ba59e24426b15bcb21e3385540, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/b192a9ef10554cc488020be2f9359663, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/3f460f0461224924b0f1af2f4da502f9, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/51081ca63d9640a6818dd51fc1feab93, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/b331d64733024be6b5cb7b71d18147bb, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/9d1b8bfc9c2e4b98b4f20a47eb55b4da, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/fc969c2f32794359b5408a7db01e8a6c, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/6b60e57b6ef448d9840f17c3fe339794, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/87d64a12054b44a7bd29db1c983ea283, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/915f1ab37e9d48098ea33b1469298c95, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/8b36c9f3e20144049115d90a3a0b4e73, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/4876f3251ea8498a997159e32135152a, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/bf2e3d352710479e8353f1f9a2ce3206, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/3dded0da65aa4ad9a2a4b05b24455f4b, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/a40e8a6d3aba45cc8ee8a4c18a8a675c, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/74ec07b6e99b469eb30d3692f1867ae1, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/d86d07f1eea24ef8a977cb4f80207878, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/a14182a9ee9a490fbde7bc619ddea4fb, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/14a0f0f5194d448c8fc333cc1187c780, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/fa98b6ad91d443eaa137a465b682b082, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/e12bdde205ec463484d0ba08f592395d, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/1935a357b64a4c9ea0b1b7d97ad96756, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/c8e230df907545b99a89de1323000233, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/1bc221f960a943bb8866dbdc32c957d8, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/dfcb9cc76e1243f6af91300190279aaa, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/2c7ef7a7567a4d55aef6e7a0ecba26d5] to archive 2024-11-23T14:26:36,847 DEBUG [StoreCloser-TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-23T14:26:36,848 DEBUG [StoreCloser-TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/37dd9dbf45fd494babf5235cce649607 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/37dd9dbf45fd494babf5235cce649607 2024-11-23T14:26:36,848 DEBUG [StoreCloser-TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/0a7b288bb2284b32999cc74acc111b66 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/0a7b288bb2284b32999cc74acc111b66 2024-11-23T14:26:36,849 DEBUG [StoreCloser-TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/3382d99ca2d64d7c8945a5a4d7d6adbf to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/3382d99ca2d64d7c8945a5a4d7d6adbf 2024-11-23T14:26:36,850 DEBUG [StoreCloser-TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/367417322d7747e283ddfbd4c74d2e3d to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/367417322d7747e283ddfbd4c74d2e3d 2024-11-23T14:26:36,851 DEBUG [StoreCloser-TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/7ca348205a0b4c2cb4173c49f2ce76f4 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/7ca348205a0b4c2cb4173c49f2ce76f4 2024-11-23T14:26:36,851 DEBUG [StoreCloser-TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/be29c0ba59e24426b15bcb21e3385540 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/be29c0ba59e24426b15bcb21e3385540 2024-11-23T14:26:36,852 DEBUG [StoreCloser-TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/b192a9ef10554cc488020be2f9359663 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/b192a9ef10554cc488020be2f9359663 2024-11-23T14:26:36,853 DEBUG [StoreCloser-TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/3f460f0461224924b0f1af2f4da502f9 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/3f460f0461224924b0f1af2f4da502f9 2024-11-23T14:26:36,854 DEBUG [StoreCloser-TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/51081ca63d9640a6818dd51fc1feab93 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/51081ca63d9640a6818dd51fc1feab93 2024-11-23T14:26:36,854 DEBUG [StoreCloser-TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/b331d64733024be6b5cb7b71d18147bb to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/b331d64733024be6b5cb7b71d18147bb 2024-11-23T14:26:36,855 DEBUG [StoreCloser-TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/9d1b8bfc9c2e4b98b4f20a47eb55b4da to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/9d1b8bfc9c2e4b98b4f20a47eb55b4da 2024-11-23T14:26:36,856 DEBUG [StoreCloser-TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/fc969c2f32794359b5408a7db01e8a6c to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/fc969c2f32794359b5408a7db01e8a6c 2024-11-23T14:26:36,857 DEBUG [StoreCloser-TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/6b60e57b6ef448d9840f17c3fe339794 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/6b60e57b6ef448d9840f17c3fe339794 2024-11-23T14:26:36,857 DEBUG [StoreCloser-TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/87d64a12054b44a7bd29db1c983ea283 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/87d64a12054b44a7bd29db1c983ea283 2024-11-23T14:26:36,858 DEBUG [StoreCloser-TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/915f1ab37e9d48098ea33b1469298c95 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/915f1ab37e9d48098ea33b1469298c95 2024-11-23T14:26:36,859 DEBUG [StoreCloser-TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/8b36c9f3e20144049115d90a3a0b4e73 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/8b36c9f3e20144049115d90a3a0b4e73 2024-11-23T14:26:36,859 DEBUG [StoreCloser-TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/4876f3251ea8498a997159e32135152a to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/4876f3251ea8498a997159e32135152a 2024-11-23T14:26:36,860 DEBUG [StoreCloser-TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/bf2e3d352710479e8353f1f9a2ce3206 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/bf2e3d352710479e8353f1f9a2ce3206 2024-11-23T14:26:36,861 DEBUG [StoreCloser-TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/3dded0da65aa4ad9a2a4b05b24455f4b to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/3dded0da65aa4ad9a2a4b05b24455f4b 2024-11-23T14:26:36,862 DEBUG [StoreCloser-TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/a40e8a6d3aba45cc8ee8a4c18a8a675c to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/a40e8a6d3aba45cc8ee8a4c18a8a675c 2024-11-23T14:26:36,862 DEBUG [StoreCloser-TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/74ec07b6e99b469eb30d3692f1867ae1 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/74ec07b6e99b469eb30d3692f1867ae1 2024-11-23T14:26:36,863 DEBUG [StoreCloser-TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/d86d07f1eea24ef8a977cb4f80207878 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/d86d07f1eea24ef8a977cb4f80207878 2024-11-23T14:26:36,864 DEBUG [StoreCloser-TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/a14182a9ee9a490fbde7bc619ddea4fb to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/a14182a9ee9a490fbde7bc619ddea4fb 2024-11-23T14:26:36,865 DEBUG [StoreCloser-TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/14a0f0f5194d448c8fc333cc1187c780 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/14a0f0f5194d448c8fc333cc1187c780 2024-11-23T14:26:36,865 DEBUG [StoreCloser-TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/fa98b6ad91d443eaa137a465b682b082 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/fa98b6ad91d443eaa137a465b682b082 2024-11-23T14:26:36,866 DEBUG [StoreCloser-TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/e12bdde205ec463484d0ba08f592395d to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/e12bdde205ec463484d0ba08f592395d 2024-11-23T14:26:36,867 DEBUG [StoreCloser-TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/1935a357b64a4c9ea0b1b7d97ad96756 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/1935a357b64a4c9ea0b1b7d97ad96756 2024-11-23T14:26:36,868 DEBUG [StoreCloser-TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/c8e230df907545b99a89de1323000233 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/c8e230df907545b99a89de1323000233 2024-11-23T14:26:36,868 DEBUG [StoreCloser-TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/1bc221f960a943bb8866dbdc32c957d8 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/1bc221f960a943bb8866dbdc32c957d8 2024-11-23T14:26:36,869 DEBUG [StoreCloser-TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/dfcb9cc76e1243f6af91300190279aaa to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/dfcb9cc76e1243f6af91300190279aaa 2024-11-23T14:26:36,870 DEBUG [StoreCloser-TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/2c7ef7a7567a4d55aef6e7a0ecba26d5 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/2c7ef7a7567a4d55aef6e7a0ecba26d5 2024-11-23T14:26:36,873 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/recovered.edits/469.seqid, newMaxSeqId=469, maxSeqId=1 2024-11-23T14:26:36,874 INFO [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. 2024-11-23T14:26:36,874 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1676): Region close journal for baf9478a6ce915eeb361816d4eed6489: Waiting for close lock at 1732371995518Running coprocessor pre-close hooks at 1732371995518Disabling compacts and flushes for region at 1732371995518Disabling writes for close at 1732371995518Obtaining lock to block concurrent updates at 1732371995518Preparing flush snapshotting stores in baf9478a6ce915eeb361816d4eed6489 at 1732371995518Finished memstore snapshotting TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1732371995518Flushing stores of TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489. at 1732371995519 (+1 ms)Flushing baf9478a6ce915eeb361816d4eed6489/A: creating writer at 1732371995519Flushing baf9478a6ce915eeb361816d4eed6489/A: appending metadata at 1732371995522 (+3 ms)Flushing baf9478a6ce915eeb361816d4eed6489/A: closing flushed file at 1732371995522Flushing baf9478a6ce915eeb361816d4eed6489/B: creating writer at 1732371995931 (+409 ms)Flushing baf9478a6ce915eeb361816d4eed6489/B: appending metadata at 1732371995933 (+2 ms)Flushing baf9478a6ce915eeb361816d4eed6489/B: closing flushed file at 1732371995933Flushing baf9478a6ce915eeb361816d4eed6489/C: creating writer at 1732371996349 (+416 ms)Flushing baf9478a6ce915eeb361816d4eed6489/C: appending metadata at 1732371996352 (+3 ms)Flushing baf9478a6ce915eeb361816d4eed6489/C: closing flushed file at 1732371996352Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5989b180: reopening flushed file at 1732371996768 (+416 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5f6ea0f0: reopening flushed file at 1732371996774 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7c3849d: reopening flushed file at 1732371996778 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for baf9478a6ce915eeb361816d4eed6489 in 1266ms, sequenceid=466, compaction requested=true at 1732371996784 (+6 ms)Writing region close event to WAL at 1732371996871 (+87 ms)Running coprocessor post-close hooks at 1732371996874 (+3 ms)Closed at 1732371996874 2024-11-23T14:26:36,875 INFO [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(157): Closed baf9478a6ce915eeb361816d4eed6489 2024-11-23T14:26:36,876 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=baf9478a6ce915eeb361816d4eed6489, regionState=CLOSED 2024-11-23T14:26:36,877 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=89, ppid=88, state=RUNNABLE, hasLock=false; CloseRegionProcedure baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 because future has completed 2024-11-23T14:26:36,879 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-23T14:26:36,879 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; CloseRegionProcedure baf9478a6ce915eeb361816d4eed6489, server=c6f7b1897123,35385,1732371902767 in 1.5120 sec 2024-11-23T14:26:36,881 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87 2024-11-23T14:26:36,881 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=baf9478a6ce915eeb361816d4eed6489, UNASSIGN in 1.5160 sec 2024-11-23T14:26:36,882 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-23T14:26:36,883 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5200 sec 2024-11-23T14:26:36,884 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732371996883"}]},"ts":"1732371996883"} 2024-11-23T14:26:36,885 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-23T14:26:36,885 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-23T14:26:36,886 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5320 sec 2024-11-23T14:26:37,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-23T14:26:37,511 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-23T14:26:37,513 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-23T14:26:37,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] procedure2.ProcedureExecutor(1139): Stored pid=90, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T14:26:37,519 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=90, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T14:26:37,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-23T14:26:37,520 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=90, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T14:26:37,522 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489 2024-11-23T14:26:37,524 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A, FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B, FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C, FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/recovered.edits] 2024-11-23T14:26:37,527 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/58ca4f4799c54b79b0d3c881b795d659 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/58ca4f4799c54b79b0d3c881b795d659 2024-11-23T14:26:37,528 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/936cc91260ed4240a4ebe080edc81dfa to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/936cc91260ed4240a4ebe080edc81dfa 2024-11-23T14:26:37,529 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/f53b6589353c4d29b9823f552349a090 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/A/f53b6589353c4d29b9823f552349a090 2024-11-23T14:26:37,531 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/c711f89b1b5c42c2a9a0adb2e7bf5cac to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/c711f89b1b5c42c2a9a0adb2e7bf5cac 2024-11-23T14:26:37,532 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/ca9cd2e8457a4a978cd33fdf03a2e790 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/ca9cd2e8457a4a978cd33fdf03a2e790 2024-11-23T14:26:37,532 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/cd63bbd3c0ef4a7baddb8ebb108b507c to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/B/cd63bbd3c0ef4a7baddb8ebb108b507c 2024-11-23T14:26:37,534 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/994402d56065441abf1cd1eceacfb935 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/994402d56065441abf1cd1eceacfb935 2024-11-23T14:26:37,535 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/be27de1eb7304afbbb03d74012c45f3d to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/be27de1eb7304afbbb03d74012c45f3d 2024-11-23T14:26:37,536 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/ee58f9163f5b48ef88899ddf4a94bd05 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/C/ee58f9163f5b48ef88899ddf4a94bd05 2024-11-23T14:26:37,538 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/recovered.edits/469.seqid to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489/recovered.edits/469.seqid 2024-11-23T14:26:37,538 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/baf9478a6ce915eeb361816d4eed6489 2024-11-23T14:26:37,539 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-23T14:26:37,540 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=90, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T14:26:37,544 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-23T14:26:37,546 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-23T14:26:37,547 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=90, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T14:26:37,547 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-23T14:26:37,547 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732371997547"}]},"ts":"9223372036854775807"} 2024-11-23T14:26:37,549 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-23T14:26:37,549 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => baf9478a6ce915eeb361816d4eed6489, NAME => 'TestAcidGuarantees,,1732371969249.baf9478a6ce915eeb361816d4eed6489.', STARTKEY => '', ENDKEY => ''}] 2024-11-23T14:26:37,549 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-23T14:26:37,549 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732371997549"}]},"ts":"9223372036854775807"} 2024-11-23T14:26:37,550 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-23T14:26:37,551 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=90, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T14:26:37,552 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 38 msec 2024-11-23T14:26:37,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-23T14:26:37,632 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-23T14:26:37,632 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-23T14:26:37,646 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=240 (was 238) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_107099228_22 at /127.0.0.1:33396 [Waiting for operation #257] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_412678638_22 at /127.0.0.1:35558 [Waiting for operation #369] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=451 (was 457), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=518 (was 468) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3405 (was 3308) - AvailableMemoryMB LEAK? - 2024-11-23T14:26:37,654 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=240, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=518, ProcessCount=11, AvailableMemoryMB=3405 2024-11-23T14:26:37,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-23T14:26:37,656 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-23T14:26:37,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] procedure2.ProcedureExecutor(1139): Stored pid=91, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-23T14:26:37,658 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-23T14:26:37,658 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:26:37,658 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 91 2024-11-23T14:26:37,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-23T14:26:37,659 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-23T14:26:37,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742184_1360 (size=959) 2024-11-23T14:26:37,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-23T14:26:37,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-23T14:26:38,068 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a 2024-11-23T14:26:38,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742185_1361 (size=53) 2024-11-23T14:26:38,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-23T14:26:38,478 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T14:26:38,479 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing fa04d9e7f6ce6ae3964ded9c7627848b, disabling compactions & flushes 2024-11-23T14:26:38,479 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:38,479 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:38,479 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. after waiting 0 ms 2024-11-23T14:26:38,479 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:38,479 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:38,479 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for fa04d9e7f6ce6ae3964ded9c7627848b: Waiting for close lock at 1732371998479Disabling compacts and flushes for region at 1732371998479Disabling writes for close at 1732371998479Writing region close event to WAL at 1732371998479Closed at 1732371998479 2024-11-23T14:26:38,481 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-23T14:26:38,482 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732371998481"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732371998481"}]},"ts":"1732371998481"} 2024-11-23T14:26:38,485 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-23T14:26:38,487 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-23T14:26:38,487 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732371998487"}]},"ts":"1732371998487"} 2024-11-23T14:26:38,488 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-23T14:26:38,489 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fa04d9e7f6ce6ae3964ded9c7627848b, ASSIGN}] 2024-11-23T14:26:38,490 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fa04d9e7f6ce6ae3964ded9c7627848b, ASSIGN 2024-11-23T14:26:38,491 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fa04d9e7f6ce6ae3964ded9c7627848b, ASSIGN; state=OFFLINE, location=c6f7b1897123,35385,1732371902767; forceNewPlan=false, retain=false 2024-11-23T14:26:38,642 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=fa04d9e7f6ce6ae3964ded9c7627848b, regionState=OPENING, regionLocation=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:38,645 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fa04d9e7f6ce6ae3964ded9c7627848b, ASSIGN because future has completed 2024-11-23T14:26:38,646 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767}] 2024-11-23T14:26:38,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-23T14:26:38,806 INFO [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:38,806 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(7752): Opening region: {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} 2024-11-23T14:26:38,807 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees fa04d9e7f6ce6ae3964ded9c7627848b 2024-11-23T14:26:38,807 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T14:26:38,807 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(7794): checking encryption for fa04d9e7f6ce6ae3964ded9c7627848b 2024-11-23T14:26:38,807 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(7797): checking classloading for fa04d9e7f6ce6ae3964ded9c7627848b 2024-11-23T14:26:38,809 INFO [StoreOpener-fa04d9e7f6ce6ae3964ded9c7627848b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region fa04d9e7f6ce6ae3964ded9c7627848b 2024-11-23T14:26:38,811 INFO [StoreOpener-fa04d9e7f6ce6ae3964ded9c7627848b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fa04d9e7f6ce6ae3964ded9c7627848b columnFamilyName A 2024-11-23T14:26:38,811 DEBUG [StoreOpener-fa04d9e7f6ce6ae3964ded9c7627848b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:26:38,812 INFO [StoreOpener-fa04d9e7f6ce6ae3964ded9c7627848b-1 {}] regionserver.HStore(327): Store=fa04d9e7f6ce6ae3964ded9c7627848b/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T14:26:38,812 INFO [StoreOpener-fa04d9e7f6ce6ae3964ded9c7627848b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region fa04d9e7f6ce6ae3964ded9c7627848b 2024-11-23T14:26:38,813 INFO [StoreOpener-fa04d9e7f6ce6ae3964ded9c7627848b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fa04d9e7f6ce6ae3964ded9c7627848b columnFamilyName B 2024-11-23T14:26:38,813 DEBUG [StoreOpener-fa04d9e7f6ce6ae3964ded9c7627848b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:26:38,814 INFO [StoreOpener-fa04d9e7f6ce6ae3964ded9c7627848b-1 {}] regionserver.HStore(327): Store=fa04d9e7f6ce6ae3964ded9c7627848b/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T14:26:38,814 INFO [StoreOpener-fa04d9e7f6ce6ae3964ded9c7627848b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region fa04d9e7f6ce6ae3964ded9c7627848b 2024-11-23T14:26:38,815 INFO [StoreOpener-fa04d9e7f6ce6ae3964ded9c7627848b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fa04d9e7f6ce6ae3964ded9c7627848b columnFamilyName C 2024-11-23T14:26:38,815 DEBUG [StoreOpener-fa04d9e7f6ce6ae3964ded9c7627848b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:26:38,815 INFO [StoreOpener-fa04d9e7f6ce6ae3964ded9c7627848b-1 {}] regionserver.HStore(327): Store=fa04d9e7f6ce6ae3964ded9c7627848b/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T14:26:38,815 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1038): replaying wal for fa04d9e7f6ce6ae3964ded9c7627848b 2024-11-23T14:26:38,816 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b 2024-11-23T14:26:38,816 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b 2024-11-23T14:26:38,817 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1048): stopping wal replay for fa04d9e7f6ce6ae3964ded9c7627848b 2024-11-23T14:26:38,817 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1060): Cleaning up temporary data for fa04d9e7f6ce6ae3964ded9c7627848b 2024-11-23T14:26:38,818 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-23T14:26:38,819 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1093): writing seq id for fa04d9e7f6ce6ae3964ded9c7627848b 2024-11-23T14:26:38,821 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T14:26:38,821 INFO [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1114): Opened fa04d9e7f6ce6ae3964ded9c7627848b; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58807368, jitterRate=-0.12370193004608154}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-23T14:26:38,821 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fa04d9e7f6ce6ae3964ded9c7627848b 2024-11-23T14:26:38,822 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1006): Region open journal for fa04d9e7f6ce6ae3964ded9c7627848b: Running coprocessor pre-open hook at 1732371998807Writing region info on filesystem at 1732371998807Initializing all the Stores at 1732371998808 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732371998809 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732371998809Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732371998809Cleaning up temporary data from old regions at 1732371998817 (+8 ms)Running coprocessor post-open hooks at 1732371998821 (+4 ms)Region opened successfully at 1732371998822 (+1 ms) 2024-11-23T14:26:38,823 INFO [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., pid=93, masterSystemTime=1732371998800 2024-11-23T14:26:38,824 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:38,824 INFO [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:38,825 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=fa04d9e7f6ce6ae3964ded9c7627848b, regionState=OPEN, openSeqNum=2, regionLocation=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:38,826 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=93, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 because future has completed 2024-11-23T14:26:38,828 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-11-23T14:26:38,828 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; OpenRegionProcedure fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 in 181 msec 2024-11-23T14:26:38,829 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-11-23T14:26:38,829 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fa04d9e7f6ce6ae3964ded9c7627848b, ASSIGN in 339 msec 2024-11-23T14:26:38,830 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-23T14:26:38,830 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732371998830"}]},"ts":"1732371998830"} 2024-11-23T14:26:38,831 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-23T14:26:38,832 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-23T14:26:38,833 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1760 sec 2024-11-23T14:26:39,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-23T14:26:39,810 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-23T14:26:39,811 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45ebd22f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:26:39,811 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6f7b1897123,38947,-1 for getting cluster id 2024-11-23T14:26:39,812 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T14:26:39,812 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c88da961-458f-4375-b9d3-275f7f90019e' 2024-11-23T14:26:39,812 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T14:26:39,812 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c88da961-458f-4375-b9d3-275f7f90019e" 2024-11-23T14:26:39,813 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60c1176e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:26:39,813 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6f7b1897123,38947,-1] 2024-11-23T14:26:39,813 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T14:26:39,813 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:26:39,813 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40644, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T14:26:39,814 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3918a791, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:26:39,814 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T14:26:39,815 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6f7b1897123,35385,1732371902767, seqNum=-1] 2024-11-23T14:26:39,815 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T14:26:39,815 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32820, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T14:26:39,816 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-23T14:26:39,817 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is c6f7b1897123,38947,1732371901705 2024-11-23T14:26:39,817 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@291a204e 2024-11-23T14:26:39,817 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-23T14:26:39,817 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40656, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-23T14:26:39,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-23T14:26:39,819 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-23T14:26:39,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] procedure2.ProcedureExecutor(1139): Stored pid=94, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-23T14:26:39,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742186_1362 (size=995) 2024-11-23T14:26:40,250 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-23T14:26:40,250 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-23T14:26:40,257 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=95, ppid=94, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-23T14:26:40,260 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fa04d9e7f6ce6ae3964ded9c7627848b, REOPEN/MOVE}] 2024-11-23T14:26:40,261 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fa04d9e7f6ce6ae3964ded9c7627848b, REOPEN/MOVE 2024-11-23T14:26:40,262 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=fa04d9e7f6ce6ae3964ded9c7627848b, regionState=CLOSING, regionLocation=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:40,263 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fa04d9e7f6ce6ae3964ded9c7627848b, REOPEN/MOVE because future has completed 2024-11-23T14:26:40,263 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-23T14:26:40,263 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=97, ppid=96, state=RUNNABLE, hasLock=false; CloseRegionProcedure fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767}] 2024-11-23T14:26:40,415 INFO [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] handler.UnassignRegionHandler(122): Close fa04d9e7f6ce6ae3964ded9c7627848b 2024-11-23T14:26:40,415 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-23T14:26:40,415 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1722): Closing fa04d9e7f6ce6ae3964ded9c7627848b, disabling compactions & flushes 2024-11-23T14:26:40,415 INFO [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:40,416 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:40,416 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. after waiting 0 ms 2024-11-23T14:26:40,416 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:40,420 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-23T14:26:40,421 INFO [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:40,421 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1676): Region close journal for fa04d9e7f6ce6ae3964ded9c7627848b: Waiting for close lock at 1732372000415Running coprocessor pre-close hooks at 1732372000415Disabling compacts and flushes for region at 1732372000415Disabling writes for close at 1732372000416 (+1 ms)Writing region close event to WAL at 1732372000416Running coprocessor post-close hooks at 1732372000420 (+4 ms)Closed at 1732372000420 2024-11-23T14:26:40,421 WARN [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegionServer(3299): Not adding moved region record: fa04d9e7f6ce6ae3964ded9c7627848b to self. 2024-11-23T14:26:40,423 INFO [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] handler.UnassignRegionHandler(157): Closed fa04d9e7f6ce6ae3964ded9c7627848b 2024-11-23T14:26:40,423 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=fa04d9e7f6ce6ae3964ded9c7627848b, regionState=CLOSED 2024-11-23T14:26:40,425 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=97, ppid=96, state=RUNNABLE, hasLock=false; CloseRegionProcedure fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 because future has completed 2024-11-23T14:26:40,426 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=97, resume processing ppid=96 2024-11-23T14:26:40,427 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, ppid=96, state=SUCCESS, hasLock=false; CloseRegionProcedure fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 in 162 msec 2024-11-23T14:26:40,427 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fa04d9e7f6ce6ae3964ded9c7627848b, REOPEN/MOVE; state=CLOSED, location=c6f7b1897123,35385,1732371902767; forceNewPlan=false, retain=true 2024-11-23T14:26:40,578 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=fa04d9e7f6ce6ae3964ded9c7627848b, regionState=OPENING, regionLocation=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:40,581 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fa04d9e7f6ce6ae3964ded9c7627848b, REOPEN/MOVE because future has completed 2024-11-23T14:26:40,582 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=96, state=RUNNABLE, hasLock=false; OpenRegionProcedure fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767}] 2024-11-23T14:26:40,742 INFO [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:40,742 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(7752): Opening region: {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} 2024-11-23T14:26:40,743 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees fa04d9e7f6ce6ae3964ded9c7627848b 2024-11-23T14:26:40,743 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T14:26:40,743 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(7794): checking encryption for fa04d9e7f6ce6ae3964ded9c7627848b 2024-11-23T14:26:40,743 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(7797): checking classloading for fa04d9e7f6ce6ae3964ded9c7627848b 2024-11-23T14:26:40,746 INFO [StoreOpener-fa04d9e7f6ce6ae3964ded9c7627848b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region fa04d9e7f6ce6ae3964ded9c7627848b 2024-11-23T14:26:40,748 INFO [StoreOpener-fa04d9e7f6ce6ae3964ded9c7627848b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fa04d9e7f6ce6ae3964ded9c7627848b columnFamilyName A 2024-11-23T14:26:40,749 DEBUG [StoreOpener-fa04d9e7f6ce6ae3964ded9c7627848b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:26:40,750 INFO [StoreOpener-fa04d9e7f6ce6ae3964ded9c7627848b-1 {}] regionserver.HStore(327): Store=fa04d9e7f6ce6ae3964ded9c7627848b/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T14:26:40,750 INFO [StoreOpener-fa04d9e7f6ce6ae3964ded9c7627848b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region fa04d9e7f6ce6ae3964ded9c7627848b 2024-11-23T14:26:40,751 INFO [StoreOpener-fa04d9e7f6ce6ae3964ded9c7627848b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fa04d9e7f6ce6ae3964ded9c7627848b columnFamilyName B 2024-11-23T14:26:40,751 DEBUG [StoreOpener-fa04d9e7f6ce6ae3964ded9c7627848b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:26:40,752 INFO [StoreOpener-fa04d9e7f6ce6ae3964ded9c7627848b-1 {}] regionserver.HStore(327): Store=fa04d9e7f6ce6ae3964ded9c7627848b/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T14:26:40,752 INFO [StoreOpener-fa04d9e7f6ce6ae3964ded9c7627848b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region fa04d9e7f6ce6ae3964ded9c7627848b 2024-11-23T14:26:40,753 INFO [StoreOpener-fa04d9e7f6ce6ae3964ded9c7627848b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fa04d9e7f6ce6ae3964ded9c7627848b columnFamilyName C 2024-11-23T14:26:40,753 DEBUG [StoreOpener-fa04d9e7f6ce6ae3964ded9c7627848b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:26:40,753 INFO [StoreOpener-fa04d9e7f6ce6ae3964ded9c7627848b-1 {}] regionserver.HStore(327): Store=fa04d9e7f6ce6ae3964ded9c7627848b/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T14:26:40,753 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1038): replaying wal for fa04d9e7f6ce6ae3964ded9c7627848b 2024-11-23T14:26:40,754 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b 2024-11-23T14:26:40,755 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b 2024-11-23T14:26:40,755 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1048): stopping wal replay for fa04d9e7f6ce6ae3964ded9c7627848b 2024-11-23T14:26:40,755 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1060): Cleaning up temporary data for fa04d9e7f6ce6ae3964ded9c7627848b 2024-11-23T14:26:40,756 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-23T14:26:40,757 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1093): writing seq id for fa04d9e7f6ce6ae3964ded9c7627848b 2024-11-23T14:26:40,758 INFO [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1114): Opened fa04d9e7f6ce6ae3964ded9c7627848b; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70416639, jitterRate=0.04928968846797943}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-23T14:26:40,758 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fa04d9e7f6ce6ae3964ded9c7627848b 2024-11-23T14:26:40,758 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1006): Region open journal for fa04d9e7f6ce6ae3964ded9c7627848b: Running coprocessor pre-open hook at 1732372000744Writing region info on filesystem at 1732372000744Initializing all the Stores at 1732372000745 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732372000745Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732372000746 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732372000746Cleaning up temporary data from old regions at 1732372000755 (+9 ms)Running coprocessor post-open hooks at 1732372000758 (+3 ms)Region opened successfully at 1732372000758 2024-11-23T14:26:40,759 INFO [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., pid=98, masterSystemTime=1732372000736 2024-11-23T14:26:40,761 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:40,761 INFO [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:40,761 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=fa04d9e7f6ce6ae3964ded9c7627848b, regionState=OPEN, openSeqNum=5, regionLocation=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:40,762 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=98, ppid=96, state=RUNNABLE, hasLock=false; OpenRegionProcedure fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 because future has completed 2024-11-23T14:26:40,765 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=96 2024-11-23T14:26:40,765 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=96, state=SUCCESS, hasLock=false; OpenRegionProcedure fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 in 181 msec 2024-11-23T14:26:40,766 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-23T14:26:40,766 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fa04d9e7f6ce6ae3964ded9c7627848b, REOPEN/MOVE in 505 msec 2024-11-23T14:26:40,768 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=95, resume processing ppid=94 2024-11-23T14:26:40,768 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, ppid=94, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 510 msec 2024-11-23T14:26:40,770 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 950 msec 2024-11-23T14:26:40,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=94 2024-11-23T14:26:40,771 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-23T14:26:40,772 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b81bdd1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:26:40,772 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6f7b1897123,38947,-1 for getting cluster id 2024-11-23T14:26:40,772 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T14:26:40,773 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c88da961-458f-4375-b9d3-275f7f90019e' 2024-11-23T14:26:40,773 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T14:26:40,773 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c88da961-458f-4375-b9d3-275f7f90019e" 2024-11-23T14:26:40,773 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b2af295, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:26:40,773 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6f7b1897123,38947,-1] 2024-11-23T14:26:40,773 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T14:26:40,773 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:26:40,774 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40682, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T14:26:40,774 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5816523a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:26:40,775 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3548d87c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:26:40,775 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6f7b1897123,38947,-1 for getting cluster id 2024-11-23T14:26:40,775 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T14:26:40,776 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c88da961-458f-4375-b9d3-275f7f90019e' 2024-11-23T14:26:40,776 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T14:26:40,776 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c88da961-458f-4375-b9d3-275f7f90019e" 2024-11-23T14:26:40,776 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@206b5891, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:26:40,776 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6f7b1897123,38947,-1] 2024-11-23T14:26:40,776 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T14:26:40,777 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:26:40,777 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40706, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T14:26:40,778 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@682f7ca1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:26:40,779 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b60a361, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:26:40,779 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6f7b1897123,38947,-1 for getting cluster id 2024-11-23T14:26:40,779 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T14:26:40,780 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c88da961-458f-4375-b9d3-275f7f90019e' 2024-11-23T14:26:40,780 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T14:26:40,780 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c88da961-458f-4375-b9d3-275f7f90019e" 2024-11-23T14:26:40,780 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31be7fe0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:26:40,780 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6f7b1897123,38947,-1] 2024-11-23T14:26:40,780 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T14:26:40,780 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:26:40,781 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40718, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T14:26:40,782 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26d1b362, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:26:40,782 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f0b8a36, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:26:40,782 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6f7b1897123,38947,-1 for getting cluster id 2024-11-23T14:26:40,782 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T14:26:40,783 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c88da961-458f-4375-b9d3-275f7f90019e' 2024-11-23T14:26:40,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T14:26:40,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c88da961-458f-4375-b9d3-275f7f90019e" 2024-11-23T14:26:40,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b2e0260, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:26:40,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6f7b1897123,38947,-1] 2024-11-23T14:26:40,784 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T14:26:40,784 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:26:40,784 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40738, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T14:26:40,785 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52481718, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:26:40,786 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21d0f7be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:26:40,786 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6f7b1897123,38947,-1 for getting cluster id 2024-11-23T14:26:40,786 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T14:26:40,787 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c88da961-458f-4375-b9d3-275f7f90019e' 2024-11-23T14:26:40,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T14:26:40,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c88da961-458f-4375-b9d3-275f7f90019e" 2024-11-23T14:26:40,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@325d83ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:26:40,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6f7b1897123,38947,-1] 2024-11-23T14:26:40,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T14:26:40,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:26:40,788 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40766, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T14:26:40,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36ef6904, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:26:40,789 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a91e16, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:26:40,789 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6f7b1897123,38947,-1 for getting cluster id 2024-11-23T14:26:40,789 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T14:26:40,790 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c88da961-458f-4375-b9d3-275f7f90019e' 2024-11-23T14:26:40,790 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T14:26:40,790 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c88da961-458f-4375-b9d3-275f7f90019e" 2024-11-23T14:26:40,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77f4f35, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:26:40,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6f7b1897123,38947,-1] 2024-11-23T14:26:40,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T14:26:40,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:26:40,792 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40796, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T14:26:40,792 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4599eae8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:26:40,793 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@aa15fd5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:26:40,793 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6f7b1897123,38947,-1 for getting cluster id 2024-11-23T14:26:40,793 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T14:26:40,794 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c88da961-458f-4375-b9d3-275f7f90019e' 2024-11-23T14:26:40,794 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T14:26:40,794 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c88da961-458f-4375-b9d3-275f7f90019e" 2024-11-23T14:26:40,794 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@637fb1d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:26:40,794 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6f7b1897123,38947,-1] 2024-11-23T14:26:40,794 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T14:26:40,794 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:26:40,795 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40814, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T14:26:40,795 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42833065, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:26:40,796 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32b68c22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:26:40,796 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6f7b1897123,38947,-1 for getting cluster id 2024-11-23T14:26:40,796 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T14:26:40,797 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c88da961-458f-4375-b9d3-275f7f90019e' 2024-11-23T14:26:40,797 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T14:26:40,797 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c88da961-458f-4375-b9d3-275f7f90019e" 2024-11-23T14:26:40,797 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48a39730, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:26:40,797 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6f7b1897123,38947,-1] 2024-11-23T14:26:40,797 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T14:26:40,798 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:26:40,798 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40838, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T14:26:40,799 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@690d72d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:26:40,799 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@390f707c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:26:40,799 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6f7b1897123,38947,-1 for getting cluster id 2024-11-23T14:26:40,799 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T14:26:40,800 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c88da961-458f-4375-b9d3-275f7f90019e' 2024-11-23T14:26:40,800 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T14:26:40,801 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c88da961-458f-4375-b9d3-275f7f90019e" 2024-11-23T14:26:40,801 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75840ac2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:26:40,801 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6f7b1897123,38947,-1] 2024-11-23T14:26:40,801 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T14:26:40,801 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:26:40,802 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40856, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T14:26:40,803 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@469812fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:26:40,803 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49ddbcc3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:26:40,803 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6f7b1897123,38947,-1 for getting cluster id 2024-11-23T14:26:40,803 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T14:26:40,804 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c88da961-458f-4375-b9d3-275f7f90019e' 2024-11-23T14:26:40,804 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T14:26:40,804 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c88da961-458f-4375-b9d3-275f7f90019e" 2024-11-23T14:26:40,804 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37988654, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:26:40,804 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6f7b1897123,38947,-1] 2024-11-23T14:26:40,804 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T14:26:40,805 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:26:40,805 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40876, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T14:26:40,806 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@195382bf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:26:40,806 DEBUG [Thread-1654 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T14:26:40,806 DEBUG [Thread-1650 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T14:26:40,807 DEBUG [Thread-1645 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T14:26:40,807 DEBUG [Thread-1639 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T14:26:40,807 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6f7b1897123,35385,1732371902767, seqNum=-1] 2024-11-23T14:26:40,807 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6f7b1897123,35385,1732371902767, seqNum=-1] 2024-11-23T14:26:40,807 DEBUG [Thread-1641 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T14:26:40,807 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T14:26:40,807 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6f7b1897123,35385,1732371902767, seqNum=-1] 2024-11-23T14:26:40,807 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6f7b1897123,35385,1732371902767, seqNum=-1] 2024-11-23T14:26:40,808 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6f7b1897123,35385,1732371902767, seqNum=-1] 2024-11-23T14:26:40,808 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T14:26:40,808 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T14:26:40,808 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T14:26:40,808 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T14:26:40,808 DEBUG [Thread-1643 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T14:26:40,808 DEBUG [Thread-1658 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T14:26:40,808 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-23T14:26:40,808 DEBUG [Thread-1647 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T14:26:40,808 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6f7b1897123,35385,1732371902767, seqNum=-1] 2024-11-23T14:26:40,809 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T14:26:40,809 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32830, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T14:26:40,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6f7b1897123,35385,1732371902767, seqNum=-1] 2024-11-23T14:26:40,809 DEBUG [Thread-1652 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T14:26:40,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T14:26:40,809 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32838, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T14:26:40,810 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6f7b1897123,35385,1732371902767, seqNum=-1] 2024-11-23T14:26:40,810 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6f7b1897123,35385,1732371902767, seqNum=-1] 2024-11-23T14:26:40,810 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T14:26:40,810 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T14:26:40,810 DEBUG [Thread-1656 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T14:26:40,810 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32848, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T14:26:40,811 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6f7b1897123,35385,1732371902767, seqNum=-1] 2024-11-23T14:26:40,811 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T14:26:40,811 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32856, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T14:26:40,811 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32882, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T14:26:40,812 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32916, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T14:26:40,812 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5] 2024-11-23T14:26:40,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-23T14:26:40,813 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32860, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T14:26:40,813 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32896, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T14:26:40,813 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5] 2024-11-23T14:26:40,813 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5] 2024-11-23T14:26:40,814 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32874, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T14:26:40,814 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32902, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T14:26:40,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-23T14:26:40,814 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5] 2024-11-23T14:26:40,814 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5] 2024-11-23T14:26:40,814 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5] 2024-11-23T14:26:40,815 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5] 2024-11-23T14:26:40,816 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T14:26:40,816 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5] 2024-11-23T14:26:40,817 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T14:26:40,817 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T14:26:40,818 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5] 2024-11-23T14:26:40,818 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5] 2024-11-23T14:26:40,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on fa04d9e7f6ce6ae3964ded9c7627848b 2024-11-23T14:26:40,831 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T14:26:40,831 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fa04d9e7f6ce6ae3964ded9c7627848b 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-23T14:26:40,864 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112300f1b1e179634fc386c36985405647ac_fa04d9e7f6ce6ae3964ded9c7627848b is 50, key is test_row_0/A:col10/1732372000818/Put/seqid=0 2024-11-23T14:26:40,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742187_1363 (size=12154) 2024-11-23T14:26:40,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-23T14:26:40,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:40,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32856 deadline: 1732372060916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:40,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:40,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32896 deadline: 1732372060916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:40,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:40,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:40,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:40,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:40,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:40,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:40,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32838 deadline: 1732372060916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:40,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:40,928 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:40,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:40,928 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:40,928 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:40,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32848 deadline: 1732372060918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:40,929 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:40,929 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:40,929 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:40,937 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:40,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32882 deadline: 1732372060927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:40,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:40,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:40,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:40,974 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-23T14:26:40,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:40,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. as already flushing 2024-11-23T14:26:40,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:40,974 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:40,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:40,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:41,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:41,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32856 deadline: 1732372061030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:41,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:41,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:41,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:41,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:41,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32896 deadline: 1732372061030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:41,037 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:41,037 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:41,037 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:41,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:41,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32838 deadline: 1732372061040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:41,045 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:41,045 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:41,045 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:41,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32848 deadline: 1732372061041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:41,045 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:41,045 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:41,045 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:41,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32882 deadline: 1732372061041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:41,045 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:41,045 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:41,045 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:41,045 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:41,045 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:41,126 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-23T14:26:41,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:41,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. as already flushing 2024-11-23T14:26:41,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:41,127 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:41,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:41,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:41,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-23T14:26:41,245 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:41,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32856 deadline: 1732372061241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:41,245 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:41,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32896 deadline: 1732372061241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:41,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:41,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:41,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:41,245 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:41,245 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:41,246 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:41,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:41,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32838 deadline: 1732372061250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:41,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:41,253 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:41,253 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:41,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32848 deadline: 1732372061250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:41,253 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:41,253 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:41,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:41,253 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:41,253 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:41,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32882 deadline: 1732372061250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:41,254 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:41,254 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:41,254 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:41,279 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-23T14:26:41,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:41,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. as already flushing 2024-11-23T14:26:41,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:41,279 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:41,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:41,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:41,282 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:26:41,288 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112300f1b1e179634fc386c36985405647ac_fa04d9e7f6ce6ae3964ded9c7627848b to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112300f1b1e179634fc386c36985405647ac_fa04d9e7f6ce6ae3964ded9c7627848b 2024-11-23T14:26:41,289 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/A/389f3cb1fc5a4c20b1deef0d5473f6cb, store: [table=TestAcidGuarantees family=A region=fa04d9e7f6ce6ae3964ded9c7627848b] 2024-11-23T14:26:41,289 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/A/389f3cb1fc5a4c20b1deef0d5473f6cb is 175, key is test_row_0/A:col10/1732372000818/Put/seqid=0 2024-11-23T14:26:41,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742188_1364 (size=30951) 2024-11-23T14:26:41,294 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/A/389f3cb1fc5a4c20b1deef0d5473f6cb 2024-11-23T14:26:41,322 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/B/fc69506ab2b048cf9019543873e0a017 is 50, key is test_row_0/B:col10/1732372000818/Put/seqid=0 2024-11-23T14:26:41,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742189_1365 (size=9657) 2024-11-23T14:26:41,328 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/B/fc69506ab2b048cf9019543873e0a017 2024-11-23T14:26:41,350 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/C/5cdba96e3153415ea6a1f22ef7dac93c is 50, key is test_row_0/C:col10/1732372000818/Put/seqid=0 2024-11-23T14:26:41,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742190_1366 (size=9657) 2024-11-23T14:26:41,431 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-23T14:26:41,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:41,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. as already flushing 2024-11-23T14:26:41,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:41,432 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:41,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:41,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:41,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-23T14:26:41,554 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:41,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32856 deadline: 1732372061550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:41,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:41,554 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:41,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:41,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:41,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32896 deadline: 1732372061550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:41,554 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:41,555 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:41,555 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:41,562 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:41,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32838 deadline: 1732372061560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:41,563 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:41,563 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:41,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32848 deadline: 1732372061561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:41,563 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:41,563 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:41,563 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:41,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32882 deadline: 1732372061561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:41,564 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:41,564 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:41,564 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:41,564 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:41,564 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:41,564 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:41,584 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-23T14:26:41,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:41,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. as already flushing 2024-11-23T14:26:41,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:41,584 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:41,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:41,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:41,737 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-23T14:26:41,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:41,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. as already flushing 2024-11-23T14:26:41,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:41,737 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:41,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:41,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:41,757 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/C/5cdba96e3153415ea6a1f22ef7dac93c 2024-11-23T14:26:41,761 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/A/389f3cb1fc5a4c20b1deef0d5473f6cb as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/389f3cb1fc5a4c20b1deef0d5473f6cb 2024-11-23T14:26:41,764 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/389f3cb1fc5a4c20b1deef0d5473f6cb, entries=150, sequenceid=16, filesize=30.2 K 2024-11-23T14:26:41,764 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/B/fc69506ab2b048cf9019543873e0a017 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/fc69506ab2b048cf9019543873e0a017 2024-11-23T14:26:41,767 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/fc69506ab2b048cf9019543873e0a017, entries=100, sequenceid=16, filesize=9.4 K 2024-11-23T14:26:41,768 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/C/5cdba96e3153415ea6a1f22ef7dac93c as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/5cdba96e3153415ea6a1f22ef7dac93c 2024-11-23T14:26:41,772 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/5cdba96e3153415ea6a1f22ef7dac93c, entries=100, sequenceid=16, filesize=9.4 K 2024-11-23T14:26:41,772 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for fa04d9e7f6ce6ae3964ded9c7627848b in 941ms, sequenceid=16, compaction requested=false 2024-11-23T14:26:41,772 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fa04d9e7f6ce6ae3964ded9c7627848b: 2024-11-23T14:26:41,891 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-23T14:26:41,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:41,891 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing fa04d9e7f6ce6ae3964ded9c7627848b 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-23T14:26:41,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411230a0ac9eaabbf4c11a9c19aa99c73c2ce_fa04d9e7f6ce6ae3964ded9c7627848b is 50, key is test_row_0/A:col10/1732372000915/Put/seqid=0 2024-11-23T14:26:41,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742191_1367 (size=12154) 2024-11-23T14:26:41,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-23T14:26:42,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on fa04d9e7f6ce6ae3964ded9c7627848b 2024-11-23T14:26:42,063 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. as already flushing 2024-11-23T14:26:42,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:42,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32856 deadline: 1732372062074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:42,078 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:42,078 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:42,078 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:42,081 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:42,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32882 deadline: 1732372062075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:42,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:42,081 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:42,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:42,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:42,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32896 deadline: 1732372062076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:42,082 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:42,082 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:42,082 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:42,084 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:42,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32838 deadline: 1732372062078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:42,085 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:42,085 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:42,085 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:42,085 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:42,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32848 deadline: 1732372062078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:42,085 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:42,085 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:42,085 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:42,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:42,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32856 deadline: 1732372062180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:42,184 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:42,184 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:42,184 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:42,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:42,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32896 deadline: 1732372062190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:42,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:42,193 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:42,193 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:42,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32838 deadline: 1732372062190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:42,193 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:42,194 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:42,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:42,194 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:42,194 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:42,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32882 deadline: 1732372062190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:42,194 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:42,194 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:42,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:42,194 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:42,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32848 deadline: 1732372062191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:42,194 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:42,194 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:42,194 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:42,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:26:42,314 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411230a0ac9eaabbf4c11a9c19aa99c73c2ce_fa04d9e7f6ce6ae3964ded9c7627848b to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411230a0ac9eaabbf4c11a9c19aa99c73c2ce_fa04d9e7f6ce6ae3964ded9c7627848b 2024-11-23T14:26:42,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/A/8344cf9e6c7746d38920b367666524f9, store: [table=TestAcidGuarantees family=A region=fa04d9e7f6ce6ae3964ded9c7627848b] 2024-11-23T14:26:42,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/A/8344cf9e6c7746d38920b367666524f9 is 175, key is test_row_0/A:col10/1732372000915/Put/seqid=0 2024-11-23T14:26:42,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742192_1368 (size=30955) 2024-11-23T14:26:42,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:42,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32856 deadline: 1732372062390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:42,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:42,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:42,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:42,402 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-23T14:26:42,406 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:42,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32896 deadline: 1732372062400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:42,406 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:42,406 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:42,406 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:42,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32838 deadline: 1732372062400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:42,406 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:42,407 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:42,407 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:42,407 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:42,407 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:42,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32882 deadline: 1732372062401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:42,407 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:42,407 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:42,407 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:42,407 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:42,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32848 deadline: 1732372062401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:42,407 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:42,407 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:42,407 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:42,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:42,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32856 deadline: 1732372062700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:42,703 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:42,703 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:42,703 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:42,713 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:42,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32896 deadline: 1732372062710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:42,714 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:42,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:42,714 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:42,714 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:42,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32882 deadline: 1732372062710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:42,714 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:42,714 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:42,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:42,714 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:42,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32838 deadline: 1732372062710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:42,715 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:42,715 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:42,715 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:42,719 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/A/8344cf9e6c7746d38920b367666524f9 2024-11-23T14:26:42,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/B/b77d2da995604fd59e03f54dd9ed9297 is 50, key is test_row_0/B:col10/1732372000915/Put/seqid=0 2024-11-23T14:26:42,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:42,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32848 deadline: 1732372062720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:42,726 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:42,726 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:42,726 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:42,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742193_1369 (size=12001) 2024-11-23T14:26:42,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-23T14:26:43,130 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/B/b77d2da995604fd59e03f54dd9ed9297 2024-11-23T14:26:43,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/C/977d1672caee41578b7c48dedea8169b is 50, key is test_row_0/C:col10/1732372000915/Put/seqid=0 2024-11-23T14:26:43,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742194_1370 (size=12001) 2024-11-23T14:26:43,145 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/C/977d1672caee41578b7c48dedea8169b 2024-11-23T14:26:43,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/A/8344cf9e6c7746d38920b367666524f9 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/8344cf9e6c7746d38920b367666524f9 2024-11-23T14:26:43,159 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/8344cf9e6c7746d38920b367666524f9, entries=150, sequenceid=40, filesize=30.2 K 2024-11-23T14:26:43,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/B/b77d2da995604fd59e03f54dd9ed9297 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/b77d2da995604fd59e03f54dd9ed9297 2024-11-23T14:26:43,163 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/b77d2da995604fd59e03f54dd9ed9297, entries=150, sequenceid=40, filesize=11.7 K 2024-11-23T14:26:43,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/C/977d1672caee41578b7c48dedea8169b as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/977d1672caee41578b7c48dedea8169b 2024-11-23T14:26:43,167 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/977d1672caee41578b7c48dedea8169b, entries=150, sequenceid=40, filesize=11.7 K 2024-11-23T14:26:43,168 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for fa04d9e7f6ce6ae3964ded9c7627848b in 1276ms, sequenceid=40, compaction requested=false 2024-11-23T14:26:43,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for fa04d9e7f6ce6ae3964ded9c7627848b: 2024-11-23T14:26:43,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:43,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-23T14:26:43,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-23T14:26:43,171 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-23T14:26:43,171 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3520 sec 2024-11-23T14:26:43,173 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 2.3640 sec 2024-11-23T14:26:43,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on fa04d9e7f6ce6ae3964ded9c7627848b 2024-11-23T14:26:43,214 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T14:26:43,214 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fa04d9e7f6ce6ae3964ded9c7627848b 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-23T14:26:43,220 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123e1d86241c93f4360ac54feaa88e680a0_fa04d9e7f6ce6ae3964ded9c7627848b is 50, key is test_row_0/A:col10/1732372002076/Put/seqid=0 2024-11-23T14:26:43,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742195_1371 (size=14594) 2024-11-23T14:26:43,224 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:26:43,227 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123e1d86241c93f4360ac54feaa88e680a0_fa04d9e7f6ce6ae3964ded9c7627848b to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123e1d86241c93f4360ac54feaa88e680a0_fa04d9e7f6ce6ae3964ded9c7627848b 2024-11-23T14:26:43,228 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/A/a3909864813b4763b7c286f25b69d532, store: [table=TestAcidGuarantees family=A region=fa04d9e7f6ce6ae3964ded9c7627848b] 2024-11-23T14:26:43,229 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/A/a3909864813b4763b7c286f25b69d532 is 175, key is test_row_0/A:col10/1732372002076/Put/seqid=0 2024-11-23T14:26:43,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742196_1372 (size=39549) 2024-11-23T14:26:43,261 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:43,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32838 deadline: 1732372063248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:43,261 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:43,261 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:43,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32896 deadline: 1732372063249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:43,262 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:43,262 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:43,262 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:43,262 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:43,262 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:43,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:43,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32882 deadline: 1732372063250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:43,262 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:43,262 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:43,262 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:43,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:43,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32856 deadline: 1732372063259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:43,263 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:43,263 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:43,263 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:43,271 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:43,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32848 deadline: 1732372063260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:43,271 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:43,271 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:43,271 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:43,371 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:43,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32838 deadline: 1732372063370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:43,372 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:43,372 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:43,372 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:43,372 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:43,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32896 deadline: 1732372063370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:43,372 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:43,372 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:43,372 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:43,372 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:43,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32882 deadline: 1732372063370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:43,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:43,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:43,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:43,373 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:43,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32856 deadline: 1732372063370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:43,373 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:43,373 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:43,373 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:43,386 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:43,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32848 deadline: 1732372063380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:43,386 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:43,386 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:43,386 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:43,586 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:43,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32838 deadline: 1732372063580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:43,586 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:43,586 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:43,586 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:43,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32896 deadline: 1732372063580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:43,586 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:43,586 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:43,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:43,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:43,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:43,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32882 deadline: 1732372063580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:43,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:43,587 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:43,587 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:43,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32856 deadline: 1732372063580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:43,587 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:43,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:43,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:43,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:43,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:43,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32848 deadline: 1732372063590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:43,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:43,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:43,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:43,633 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=53, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/A/a3909864813b4763b7c286f25b69d532 2024-11-23T14:26:43,638 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/B/cc893cfd2fb548b3a42a1047b872a8c3 is 50, key is test_row_0/B:col10/1732372002076/Put/seqid=0 2024-11-23T14:26:43,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742197_1373 (size=12001) 2024-11-23T14:26:43,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:43,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32838 deadline: 1732372063891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:43,892 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:43,892 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:43,892 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:43,892 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:43,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32896 deadline: 1732372063891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:43,892 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:43,892 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:43,892 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:43,892 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:43,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32882 deadline: 1732372063891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:43,892 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:43,892 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:43,892 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:43,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:43,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32856 deadline: 1732372063900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:43,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:43,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:43,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:43,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:43,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32848 deadline: 1732372063900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:43,905 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:43,905 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:43,905 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:44,045 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/B/cc893cfd2fb548b3a42a1047b872a8c3 2024-11-23T14:26:44,050 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/C/f8e58cc8ce944f50aee652d8df260ec1 is 50, key is test_row_0/C:col10/1732372002076/Put/seqid=0 2024-11-23T14:26:44,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742198_1374 (size=12001) 2024-11-23T14:26:44,406 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:44,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32838 deadline: 1732372064400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:44,406 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:44,406 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:44,406 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:44,406 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:44,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32896 deadline: 1732372064400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:44,406 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:44,407 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:44,407 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:44,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:44,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32882 deadline: 1732372064400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:44,407 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:44,407 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:44,407 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:44,417 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:44,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32856 deadline: 1732372064410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:44,417 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:44,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32848 deadline: 1732372064410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:44,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:44,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:44,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:44,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:44,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:44,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:44,455 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/C/f8e58cc8ce944f50aee652d8df260ec1 2024-11-23T14:26:44,459 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/A/a3909864813b4763b7c286f25b69d532 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/a3909864813b4763b7c286f25b69d532 2024-11-23T14:26:44,462 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/a3909864813b4763b7c286f25b69d532, entries=200, sequenceid=53, filesize=38.6 K 2024-11-23T14:26:44,463 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/B/cc893cfd2fb548b3a42a1047b872a8c3 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/cc893cfd2fb548b3a42a1047b872a8c3 2024-11-23T14:26:44,465 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/cc893cfd2fb548b3a42a1047b872a8c3, entries=150, sequenceid=53, filesize=11.7 K 2024-11-23T14:26:44,466 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/C/f8e58cc8ce944f50aee652d8df260ec1 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/f8e58cc8ce944f50aee652d8df260ec1 2024-11-23T14:26:44,468 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/f8e58cc8ce944f50aee652d8df260ec1, entries=150, sequenceid=53, filesize=11.7 K 2024-11-23T14:26:44,469 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for fa04d9e7f6ce6ae3964ded9c7627848b in 1255ms, sequenceid=53, compaction requested=true 2024-11-23T14:26:44,469 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fa04d9e7f6ce6ae3964ded9c7627848b: 2024-11-23T14:26:44,469 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:26:44,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fa04d9e7f6ce6ae3964ded9c7627848b:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T14:26:44,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:26:44,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fa04d9e7f6ce6ae3964ded9c7627848b:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T14:26:44,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:26:44,470 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:26:44,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fa04d9e7f6ce6ae3964ded9c7627848b:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T14:26:44,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:26:44,470 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101455 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:26:44,470 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1541): fa04d9e7f6ce6ae3964ded9c7627848b/A is initiating minor compaction (all files) 2024-11-23T14:26:44,470 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fa04d9e7f6ce6ae3964ded9c7627848b/A in TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:44,470 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/389f3cb1fc5a4c20b1deef0d5473f6cb, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/8344cf9e6c7746d38920b367666524f9, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/a3909864813b4763b7c286f25b69d532] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp, totalSize=99.1 K 2024-11-23T14:26:44,470 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:44,471 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. files: [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/389f3cb1fc5a4c20b1deef0d5473f6cb, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/8344cf9e6c7746d38920b367666524f9, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/a3909864813b4763b7c286f25b69d532] 2024-11-23T14:26:44,471 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:26:44,471 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 389f3cb1fc5a4c20b1deef0d5473f6cb, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732372000818 2024-11-23T14:26:44,471 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1541): fa04d9e7f6ce6ae3964ded9c7627848b/B is initiating minor compaction (all files) 2024-11-23T14:26:44,471 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fa04d9e7f6ce6ae3964ded9c7627848b/B in TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:44,471 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/fc69506ab2b048cf9019543873e0a017, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/b77d2da995604fd59e03f54dd9ed9297, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/cc893cfd2fb548b3a42a1047b872a8c3] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp, totalSize=32.9 K 2024-11-23T14:26:44,471 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8344cf9e6c7746d38920b367666524f9, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732372000856 2024-11-23T14:26:44,471 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting fc69506ab2b048cf9019543873e0a017, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732372000818 2024-11-23T14:26:44,471 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting a3909864813b4763b7c286f25b69d532, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732372002074 2024-11-23T14:26:44,471 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting b77d2da995604fd59e03f54dd9ed9297, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732372000856 2024-11-23T14:26:44,472 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting cc893cfd2fb548b3a42a1047b872a8c3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732372002075 2024-11-23T14:26:44,475 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fa04d9e7f6ce6ae3964ded9c7627848b] 2024-11-23T14:26:44,478 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fa04d9e7f6ce6ae3964ded9c7627848b#B#compaction#325 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:26:44,479 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/B/a897475d1eea46c29c3ca2cd1c3bc7be is 50, key is test_row_0/B:col10/1732372002076/Put/seqid=0 2024-11-23T14:26:44,480 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241123e54a3184c1a64445900d777501f2f779_fa04d9e7f6ce6ae3964ded9c7627848b store=[table=TestAcidGuarantees family=A region=fa04d9e7f6ce6ae3964ded9c7627848b] 2024-11-23T14:26:44,481 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241123e54a3184c1a64445900d777501f2f779_fa04d9e7f6ce6ae3964ded9c7627848b, store=[table=TestAcidGuarantees family=A region=fa04d9e7f6ce6ae3964ded9c7627848b] 2024-11-23T14:26:44,481 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123e54a3184c1a64445900d777501f2f779_fa04d9e7f6ce6ae3964ded9c7627848b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fa04d9e7f6ce6ae3964ded9c7627848b] 2024-11-23T14:26:44,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742199_1375 (size=12104) 2024-11-23T14:26:44,489 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/B/a897475d1eea46c29c3ca2cd1c3bc7be as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/a897475d1eea46c29c3ca2cd1c3bc7be 2024-11-23T14:26:44,494 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fa04d9e7f6ce6ae3964ded9c7627848b/B of fa04d9e7f6ce6ae3964ded9c7627848b into a897475d1eea46c29c3ca2cd1c3bc7be(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:26:44,494 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fa04d9e7f6ce6ae3964ded9c7627848b: 2024-11-23T14:26:44,494 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., storeName=fa04d9e7f6ce6ae3964ded9c7627848b/B, priority=13, startTime=1732372004470; duration=0sec 2024-11-23T14:26:44,494 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:26:44,494 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fa04d9e7f6ce6ae3964ded9c7627848b:B 2024-11-23T14:26:44,494 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:26:44,495 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:26:44,495 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1541): fa04d9e7f6ce6ae3964ded9c7627848b/C is initiating minor compaction (all files) 2024-11-23T14:26:44,495 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fa04d9e7f6ce6ae3964ded9c7627848b/C in TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:44,495 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/5cdba96e3153415ea6a1f22ef7dac93c, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/977d1672caee41578b7c48dedea8169b, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/f8e58cc8ce944f50aee652d8df260ec1] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp, totalSize=32.9 K 2024-11-23T14:26:44,496 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 5cdba96e3153415ea6a1f22ef7dac93c, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732372000818 2024-11-23T14:26:44,496 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 977d1672caee41578b7c48dedea8169b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732372000856 2024-11-23T14:26:44,496 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting f8e58cc8ce944f50aee652d8df260ec1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732372002075 2024-11-23T14:26:44,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742200_1376 (size=4469) 2024-11-23T14:26:44,499 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fa04d9e7f6ce6ae3964ded9c7627848b#A#compaction#324 average throughput is 1.02 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:26:44,500 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/A/85e7b9427e124a6e93e8d5e1b7787e2f is 175, key is test_row_0/A:col10/1732372002076/Put/seqid=0 2024-11-23T14:26:44,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742201_1377 (size=31058) 2024-11-23T14:26:44,511 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fa04d9e7f6ce6ae3964ded9c7627848b#C#compaction#326 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:26:44,511 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/C/61f36d55bb0949948b9f763819380aab is 50, key is test_row_0/C:col10/1732372002076/Put/seqid=0 2024-11-23T14:26:44,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742202_1378 (size=12104) 2024-11-23T14:26:44,518 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/A/85e7b9427e124a6e93e8d5e1b7787e2f as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/85e7b9427e124a6e93e8d5e1b7787e2f 2024-11-23T14:26:44,523 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/C/61f36d55bb0949948b9f763819380aab as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/61f36d55bb0949948b9f763819380aab 2024-11-23T14:26:44,524 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fa04d9e7f6ce6ae3964ded9c7627848b/A of fa04d9e7f6ce6ae3964ded9c7627848b into 85e7b9427e124a6e93e8d5e1b7787e2f(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:26:44,524 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fa04d9e7f6ce6ae3964ded9c7627848b: 2024-11-23T14:26:44,524 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., storeName=fa04d9e7f6ce6ae3964ded9c7627848b/A, priority=13, startTime=1732372004469; duration=0sec 2024-11-23T14:26:44,524 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:26:44,524 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fa04d9e7f6ce6ae3964ded9c7627848b:A 2024-11-23T14:26:44,528 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fa04d9e7f6ce6ae3964ded9c7627848b/C of fa04d9e7f6ce6ae3964ded9c7627848b into 61f36d55bb0949948b9f763819380aab(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:26:44,528 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fa04d9e7f6ce6ae3964ded9c7627848b: 2024-11-23T14:26:44,528 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., storeName=fa04d9e7f6ce6ae3964ded9c7627848b/C, priority=13, startTime=1732372004470; duration=0sec 2024-11-23T14:26:44,528 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:26:44,528 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fa04d9e7f6ce6ae3964ded9c7627848b:C 2024-11-23T14:26:44,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-23T14:26:44,970 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T14:26:44,971 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-23T14:26:44,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-23T14:26:44,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-23T14:26:44,972 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T14:26:44,973 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T14:26:44,973 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T14:26:45,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-23T14:26:45,124 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-23T14:26:45,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:45,125 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing fa04d9e7f6ce6ae3964ded9c7627848b 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-23T14:26:45,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411234e6225dfc4744cd0b7a4c64fed02a4b1_fa04d9e7f6ce6ae3964ded9c7627848b is 50, key is test_row_0/A:col10/1732372003247/Put/seqid=0 2024-11-23T14:26:45,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742203_1379 (size=12154) 2024-11-23T14:26:45,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:26:45,150 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411234e6225dfc4744cd0b7a4c64fed02a4b1_fa04d9e7f6ce6ae3964ded9c7627848b to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411234e6225dfc4744cd0b7a4c64fed02a4b1_fa04d9e7f6ce6ae3964ded9c7627848b 2024-11-23T14:26:45,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/A/8e9828f390ed48c18def9091b530da6b, store: [table=TestAcidGuarantees family=A region=fa04d9e7f6ce6ae3964ded9c7627848b] 2024-11-23T14:26:45,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/A/8e9828f390ed48c18def9091b530da6b is 175, key is test_row_0/A:col10/1732372003247/Put/seqid=0 2024-11-23T14:26:45,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742204_1380 (size=30955) 2024-11-23T14:26:45,157 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=79, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/A/8e9828f390ed48c18def9091b530da6b 2024-11-23T14:26:45,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/B/f4500ed0cbc94b57aefafb5e08e6f5a2 is 50, key is test_row_0/B:col10/1732372003247/Put/seqid=0 2024-11-23T14:26:45,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742205_1381 (size=12001) 2024-11-23T14:26:45,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-23T14:26:45,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on fa04d9e7f6ce6ae3964ded9c7627848b 2024-11-23T14:26:45,413 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. as already flushing 2024-11-23T14:26:45,438 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:45,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32838 deadline: 1732372065431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:45,438 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:45,438 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:45,438 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:45,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:45,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32896 deadline: 1732372065436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:45,440 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:45,440 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:45,440 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:45,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:45,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32882 deadline: 1732372065437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:45,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:45,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:45,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:45,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:45,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32856 deadline: 1732372065438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:45,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:45,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:45,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:45,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:45,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32848 deadline: 1732372065438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:45,445 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:45,445 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:45,445 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:45,546 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:45,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32838 deadline: 1732372065540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:45,547 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:45,547 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:45,547 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:45,556 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:45,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32896 deadline: 1732372065550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:45,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:45,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:45,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:45,556 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:45,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32882 deadline: 1732372065550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:45,557 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:45,557 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:45,557 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:45,557 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:45,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32856 deadline: 1732372065550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:45,557 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:45,557 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:45,557 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:45,557 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:45,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32848 deadline: 1732372065550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:45,557 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:45,557 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:45,557 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:45,571 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/B/f4500ed0cbc94b57aefafb5e08e6f5a2 2024-11-23T14:26:45,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/C/4e2c618d910b4d4fa7e1da0974b5708f is 50, key is test_row_0/C:col10/1732372003247/Put/seqid=0 2024-11-23T14:26:45,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742206_1382 (size=12001) 2024-11-23T14:26:45,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-23T14:26:45,754 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:45,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32838 deadline: 1732372065750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:45,755 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:45,755 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:45,755 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:45,764 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:45,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32896 deadline: 1732372065760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:45,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:45,764 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:45,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:45,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:45,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32882 deadline: 1732372065760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:45,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:45,765 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:45,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32856 deadline: 1732372065760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:45,765 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:45,765 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:45,765 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:45,765 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:45,765 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:45,774 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:45,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32848 deadline: 1732372065770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:45,775 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:45,775 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:45,775 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:45,982 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/C/4e2c618d910b4d4fa7e1da0974b5708f 2024-11-23T14:26:45,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/A/8e9828f390ed48c18def9091b530da6b as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/8e9828f390ed48c18def9091b530da6b 2024-11-23T14:26:45,994 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/8e9828f390ed48c18def9091b530da6b, entries=150, sequenceid=79, filesize=30.2 K 2024-11-23T14:26:45,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/B/f4500ed0cbc94b57aefafb5e08e6f5a2 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/f4500ed0cbc94b57aefafb5e08e6f5a2 2024-11-23T14:26:46,000 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/f4500ed0cbc94b57aefafb5e08e6f5a2, entries=150, sequenceid=79, filesize=11.7 K 2024-11-23T14:26:46,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/C/4e2c618d910b4d4fa7e1da0974b5708f as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/4e2c618d910b4d4fa7e1da0974b5708f 2024-11-23T14:26:46,010 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/4e2c618d910b4d4fa7e1da0974b5708f, entries=150, sequenceid=79, filesize=11.7 K 2024-11-23T14:26:46,011 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for fa04d9e7f6ce6ae3964ded9c7627848b in 886ms, sequenceid=79, compaction requested=false 2024-11-23T14:26:46,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for fa04d9e7f6ce6ae3964ded9c7627848b: 2024-11-23T14:26:46,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:46,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-23T14:26:46,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-23T14:26:46,017 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-23T14:26:46,017 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0400 sec 2024-11-23T14:26:46,019 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 1.0470 sec 2024-11-23T14:26:46,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on fa04d9e7f6ce6ae3964ded9c7627848b 2024-11-23T14:26:46,065 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T14:26:46,065 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fa04d9e7f6ce6ae3964ded9c7627848b 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-23T14:26:46,082 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123da270fc84e77450b8a7fdc825c3692cb_fa04d9e7f6ce6ae3964ded9c7627848b is 50, key is test_row_0/A:col10/1732372005417/Put/seqid=0 2024-11-23T14:26:46,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-23T14:26:46,111 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T14:26:46,112 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-23T14:26:46,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742207_1383 (size=14594) 2024-11-23T14:26:46,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-23T14:26:46,114 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:26:46,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-23T14:26:46,114 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T14:26:46,115 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T14:26:46,115 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T14:26:46,119 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123da270fc84e77450b8a7fdc825c3692cb_fa04d9e7f6ce6ae3964ded9c7627848b to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123da270fc84e77450b8a7fdc825c3692cb_fa04d9e7f6ce6ae3964ded9c7627848b 2024-11-23T14:26:46,120 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/A/ffc54d5f6cd54cedb9c877da92d2e89b, store: [table=TestAcidGuarantees family=A region=fa04d9e7f6ce6ae3964ded9c7627848b] 2024-11-23T14:26:46,121 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/A/ffc54d5f6cd54cedb9c877da92d2e89b is 175, key is test_row_0/A:col10/1732372005417/Put/seqid=0 2024-11-23T14:26:46,132 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:46,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32838 deadline: 1732372066116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:46,132 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:46,132 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:46,132 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:46,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:46,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32896 deadline: 1732372066115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:46,135 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:46,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:46,135 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:46,135 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:46,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32882 deadline: 1732372066120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:46,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:46,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32856 deadline: 1732372066120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:46,135 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:46,135 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:46,135 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:46,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:46,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:46,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:46,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32848 deadline: 1732372066120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:46,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:46,136 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:46,136 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:46,136 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:46,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742208_1384 (size=39549) 2024-11-23T14:26:46,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-23T14:26:46,248 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:46,248 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:46,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32838 deadline: 1732372066240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:46,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32882 deadline: 1732372066240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:46,249 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:46,249 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:46,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32896 deadline: 1732372066240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:46,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32856 deadline: 1732372066240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:46,249 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:46,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:46,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32848 deadline: 1732372066241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:46,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:46,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:46,249 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:46,250 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:46,250 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:46,251 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:46,251 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:46,251 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:46,251 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:46,251 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:46,251 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:46,251 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:46,251 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:46,251 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:46,271 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-23T14:26:46,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:46,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. as already flushing 2024-11-23T14:26:46,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:46,271 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:46,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:46,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:46,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-23T14:26:46,442 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-23T14:26:46,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:46,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. as already flushing 2024-11-23T14:26:46,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:46,442 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:46,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:46,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:46,473 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:46,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32856 deadline: 1732372066461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:46,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:46,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:46,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:46,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:46,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32848 deadline: 1732372066461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:46,476 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:46,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:46,476 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:46,476 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:46,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32882 deadline: 1732372066461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:46,476 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:46,476 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:46,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:46,477 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:46,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32838 deadline: 1732372066461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:46,477 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:46,477 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:46,477 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:46,477 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:46,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32896 deadline: 1732372066461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:46,477 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:46,477 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:46,477 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:46,545 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=94, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/A/ffc54d5f6cd54cedb9c877da92d2e89b 2024-11-23T14:26:46,554 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/B/313987ad62dd4842a7011c9fb47c05b7 is 50, key is test_row_0/B:col10/1732372005417/Put/seqid=0 2024-11-23T14:26:46,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742209_1385 (size=12001) 2024-11-23T14:26:46,596 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-23T14:26:46,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:46,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. as already flushing 2024-11-23T14:26:46,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:46,597 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:46,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:46,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:46,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-23T14:26:46,750 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-23T14:26:46,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:46,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. as already flushing 2024-11-23T14:26:46,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:46,750 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:46,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:46,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:46,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:46,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32856 deadline: 1732372066780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:46,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:46,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:46,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:46,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:46,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32882 deadline: 1732372066780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:46,786 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:46,786 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:46,786 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:46,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:46,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32848 deadline: 1732372066780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:46,787 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:46,787 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:46,787 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:46,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:46,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32838 deadline: 1732372066780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:46,791 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:46,791 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:46,791 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:46,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:46,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32896 deadline: 1732372066790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:46,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:46,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:46,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:46,903 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-23T14:26:46,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:46,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. as already flushing 2024-11-23T14:26:46,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:46,903 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:46,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:46,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:46,963 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/B/313987ad62dd4842a7011c9fb47c05b7 2024-11-23T14:26:46,994 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/C/18a624705a254bf99c7c8e7bbea86069 is 50, key is test_row_0/C:col10/1732372005417/Put/seqid=0 2024-11-23T14:26:47,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742210_1386 (size=12001) 2024-11-23T14:26:47,016 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/C/18a624705a254bf99c7c8e7bbea86069 2024-11-23T14:26:47,020 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/A/ffc54d5f6cd54cedb9c877da92d2e89b as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/ffc54d5f6cd54cedb9c877da92d2e89b 2024-11-23T14:26:47,027 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/ffc54d5f6cd54cedb9c877da92d2e89b, entries=200, sequenceid=94, filesize=38.6 K 2024-11-23T14:26:47,028 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/B/313987ad62dd4842a7011c9fb47c05b7 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/313987ad62dd4842a7011c9fb47c05b7 2024-11-23T14:26:47,032 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/313987ad62dd4842a7011c9fb47c05b7, entries=150, sequenceid=94, filesize=11.7 K 2024-11-23T14:26:47,033 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/C/18a624705a254bf99c7c8e7bbea86069 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/18a624705a254bf99c7c8e7bbea86069 2024-11-23T14:26:47,041 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/18a624705a254bf99c7c8e7bbea86069, entries=150, sequenceid=94, filesize=11.7 K 2024-11-23T14:26:47,042 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for fa04d9e7f6ce6ae3964ded9c7627848b in 977ms, sequenceid=94, compaction requested=true 2024-11-23T14:26:47,043 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fa04d9e7f6ce6ae3964ded9c7627848b: 2024-11-23T14:26:47,043 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:26:47,043 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fa04d9e7f6ce6ae3964ded9c7627848b:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T14:26:47,044 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:26:47,044 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1541): fa04d9e7f6ce6ae3964ded9c7627848b/A is initiating minor compaction (all files) 2024-11-23T14:26:47,044 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fa04d9e7f6ce6ae3964ded9c7627848b/A in TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:47,044 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/85e7b9427e124a6e93e8d5e1b7787e2f, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/8e9828f390ed48c18def9091b530da6b, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/ffc54d5f6cd54cedb9c877da92d2e89b] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp, totalSize=99.2 K 2024-11-23T14:26:47,044 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:47,044 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. files: [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/85e7b9427e124a6e93e8d5e1b7787e2f, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/8e9828f390ed48c18def9091b530da6b, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/ffc54d5f6cd54cedb9c877da92d2e89b] 2024-11-23T14:26:47,047 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:26:47,047 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 85e7b9427e124a6e93e8d5e1b7787e2f, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732372002075 2024-11-23T14:26:47,047 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:26:47,047 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8e9828f390ed48c18def9091b530da6b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1732372003247 2024-11-23T14:26:47,048 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:26:47,048 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1541): fa04d9e7f6ce6ae3964ded9c7627848b/B is initiating minor compaction (all files) 2024-11-23T14:26:47,048 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fa04d9e7f6ce6ae3964ded9c7627848b/B in TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:47,048 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/a897475d1eea46c29c3ca2cd1c3bc7be, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/f4500ed0cbc94b57aefafb5e08e6f5a2, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/313987ad62dd4842a7011c9fb47c05b7] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp, totalSize=35.3 K 2024-11-23T14:26:47,048 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting ffc54d5f6cd54cedb9c877da92d2e89b, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732372005417 2024-11-23T14:26:47,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fa04d9e7f6ce6ae3964ded9c7627848b:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T14:26:47,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:26:47,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fa04d9e7f6ce6ae3964ded9c7627848b:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T14:26:47,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:26:47,048 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting a897475d1eea46c29c3ca2cd1c3bc7be, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732372002075 2024-11-23T14:26:47,049 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting f4500ed0cbc94b57aefafb5e08e6f5a2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1732372003247 2024-11-23T14:26:47,049 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 313987ad62dd4842a7011c9fb47c05b7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732372005417 2024-11-23T14:26:47,056 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-23T14:26:47,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:47,056 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing fa04d9e7f6ce6ae3964ded9c7627848b 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-23T14:26:47,058 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fa04d9e7f6ce6ae3964ded9c7627848b] 2024-11-23T14:26:47,064 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fa04d9e7f6ce6ae3964ded9c7627848b#B#compaction#334 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:26:47,064 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/B/419ea993d355482ab74361efef7ae7ac is 50, key is test_row_0/B:col10/1732372005417/Put/seqid=0 2024-11-23T14:26:47,069 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241123d7dea089862d4abc8687c9b4298095a3_fa04d9e7f6ce6ae3964ded9c7627848b store=[table=TestAcidGuarantees family=A region=fa04d9e7f6ce6ae3964ded9c7627848b] 2024-11-23T14:26:47,071 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241123d7dea089862d4abc8687c9b4298095a3_fa04d9e7f6ce6ae3964ded9c7627848b, store=[table=TestAcidGuarantees family=A region=fa04d9e7f6ce6ae3964ded9c7627848b] 2024-11-23T14:26:47,071 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123d7dea089862d4abc8687c9b4298095a3_fa04d9e7f6ce6ae3964ded9c7627848b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fa04d9e7f6ce6ae3964ded9c7627848b] 2024-11-23T14:26:47,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123eaab989d68df4fd089421aa8204830bb_fa04d9e7f6ce6ae3964ded9c7627848b is 50, key is test_row_0/A:col10/1732372006115/Put/seqid=0 2024-11-23T14:26:47,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742211_1387 (size=12207) 2024-11-23T14:26:47,129 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/B/419ea993d355482ab74361efef7ae7ac as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/419ea993d355482ab74361efef7ae7ac 2024-11-23T14:26:47,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742212_1388 (size=4469) 2024-11-23T14:26:47,137 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fa04d9e7f6ce6ae3964ded9c7627848b/B of fa04d9e7f6ce6ae3964ded9c7627848b into 419ea993d355482ab74361efef7ae7ac(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:26:47,137 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fa04d9e7f6ce6ae3964ded9c7627848b: 2024-11-23T14:26:47,138 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., storeName=fa04d9e7f6ce6ae3964ded9c7627848b/B, priority=13, startTime=1732372007047; duration=0sec 2024-11-23T14:26:47,138 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:26:47,138 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fa04d9e7f6ce6ae3964ded9c7627848b:B 2024-11-23T14:26:47,138 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:26:47,139 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fa04d9e7f6ce6ae3964ded9c7627848b#A#compaction#333 average throughput is 0.30 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:26:47,139 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:26:47,139 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1541): fa04d9e7f6ce6ae3964ded9c7627848b/C is initiating minor compaction (all files) 2024-11-23T14:26:47,139 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fa04d9e7f6ce6ae3964ded9c7627848b/C in TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:47,140 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/61f36d55bb0949948b9f763819380aab, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/4e2c618d910b4d4fa7e1da0974b5708f, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/18a624705a254bf99c7c8e7bbea86069] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp, totalSize=35.3 K 2024-11-23T14:26:47,140 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/A/615a4c27ae5d411987c99d2ca1199948 is 175, key is test_row_0/A:col10/1732372005417/Put/seqid=0 2024-11-23T14:26:47,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742213_1389 (size=12154) 2024-11-23T14:26:47,141 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 61f36d55bb0949948b9f763819380aab, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732372002075 2024-11-23T14:26:47,141 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e2c618d910b4d4fa7e1da0974b5708f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1732372003247 2024-11-23T14:26:47,142 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 18a624705a254bf99c7c8e7bbea86069, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732372005417 2024-11-23T14:26:47,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742214_1390 (size=31161) 2024-11-23T14:26:47,154 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fa04d9e7f6ce6ae3964ded9c7627848b#C#compaction#336 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:26:47,156 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/C/c3a02fdf495c45c19d9d5ba2b579e842 is 50, key is test_row_0/C:col10/1732372005417/Put/seqid=0 2024-11-23T14:26:47,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742215_1391 (size=12207) 2024-11-23T14:26:47,173 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/C/c3a02fdf495c45c19d9d5ba2b579e842 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/c3a02fdf495c45c19d9d5ba2b579e842 2024-11-23T14:26:47,179 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fa04d9e7f6ce6ae3964ded9c7627848b/C of fa04d9e7f6ce6ae3964ded9c7627848b into c3a02fdf495c45c19d9d5ba2b579e842(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:26:47,179 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fa04d9e7f6ce6ae3964ded9c7627848b: 2024-11-23T14:26:47,179 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., storeName=fa04d9e7f6ce6ae3964ded9c7627848b/C, priority=13, startTime=1732372007048; duration=0sec 2024-11-23T14:26:47,179 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:26:47,179 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fa04d9e7f6ce6ae3964ded9c7627848b:C 2024-11-23T14:26:47,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-23T14:26:47,297 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. as already flushing 2024-11-23T14:26:47,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on fa04d9e7f6ce6ae3964ded9c7627848b 2024-11-23T14:26:47,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:47,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32838 deadline: 1732372067315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:47,327 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:47,327 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:47,327 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:47,327 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:47,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32856 deadline: 1732372067321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:47,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:47,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:47,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:47,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:47,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32882 deadline: 1732372067323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:47,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:47,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:47,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:47,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:47,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32896 deadline: 1732372067324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:47,338 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:47,338 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:47,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32848 deadline: 1732372067325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:47,338 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:47,338 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:47,338 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:47,338 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:47,338 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:47,437 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:47,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32838 deadline: 1732372067430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:47,440 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:47,440 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:47,440 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:47,450 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:47,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32896 deadline: 1732372067441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:47,450 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:47,450 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:47,450 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:47,450 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:47,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32882 deadline: 1732372067441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:47,451 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:47,451 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:47,451 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:47,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32848 deadline: 1732372067441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:47,451 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:47,451 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:47,451 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:47,451 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:47,451 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:47,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32856 deadline: 1732372067441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:47,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:47,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:47,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:47,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:26:47,545 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123eaab989d68df4fd089421aa8204830bb_fa04d9e7f6ce6ae3964ded9c7627848b to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123eaab989d68df4fd089421aa8204830bb_fa04d9e7f6ce6ae3964ded9c7627848b 2024-11-23T14:26:47,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/A/bff842b8b05b4f7eb17ed849e8a5f85b, store: [table=TestAcidGuarantees family=A region=fa04d9e7f6ce6ae3964ded9c7627848b] 2024-11-23T14:26:47,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/A/bff842b8b05b4f7eb17ed849e8a5f85b is 175, key is test_row_0/A:col10/1732372006115/Put/seqid=0 2024-11-23T14:26:47,557 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/A/615a4c27ae5d411987c99d2ca1199948 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/615a4c27ae5d411987c99d2ca1199948 2024-11-23T14:26:47,561 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fa04d9e7f6ce6ae3964ded9c7627848b/A of fa04d9e7f6ce6ae3964ded9c7627848b into 615a4c27ae5d411987c99d2ca1199948(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:26:47,561 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fa04d9e7f6ce6ae3964ded9c7627848b: 2024-11-23T14:26:47,561 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., storeName=fa04d9e7f6ce6ae3964ded9c7627848b/A, priority=13, startTime=1732372007043; duration=0sec 2024-11-23T14:26:47,562 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:26:47,562 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fa04d9e7f6ce6ae3964ded9c7627848b:A 2024-11-23T14:26:47,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742216_1392 (size=30955) 2024-11-23T14:26:47,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:47,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32838 deadline: 1732372067650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:47,653 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:47,653 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:47,653 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:47,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:47,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32882 deadline: 1732372067660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:47,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:47,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:47,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:47,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:47,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32848 deadline: 1732372067660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:47,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:47,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:47,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:47,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:47,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32856 deadline: 1732372067662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:47,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:47,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:47,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:47,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:47,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32896 deadline: 1732372067662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:47,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:47,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:47,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:47,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:47,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32838 deadline: 1732372067960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:47,964 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:47,965 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:47,965 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:47,966 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=115, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/A/bff842b8b05b4f7eb17ed849e8a5f85b 2024-11-23T14:26:47,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/B/dd50874d686941e9972d021dc9a869d6 is 50, key is test_row_0/B:col10/1732372006115/Put/seqid=0 2024-11-23T14:26:47,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:47,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32856 deadline: 1732372067971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:47,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:47,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:47,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:47,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:47,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32882 deadline: 1732372067971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:47,975 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:47,975 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:47,975 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:47,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:47,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32896 deadline: 1732372067971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:47,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:47,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:47,976 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:47,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:47,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32848 deadline: 1732372067971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:47,977 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:47,977 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:47,977 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:47,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742217_1393 (size=12001) 2024-11-23T14:26:48,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-23T14:26:48,396 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/B/dd50874d686941e9972d021dc9a869d6 2024-11-23T14:26:48,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/C/73d8814609584989a07281c06edc12f1 is 50, key is test_row_0/C:col10/1732372006115/Put/seqid=0 2024-11-23T14:26:48,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742218_1394 (size=12001) 2024-11-23T14:26:48,453 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/C/73d8814609584989a07281c06edc12f1 2024-11-23T14:26:48,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/A/bff842b8b05b4f7eb17ed849e8a5f85b as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/bff842b8b05b4f7eb17ed849e8a5f85b 2024-11-23T14:26:48,466 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/bff842b8b05b4f7eb17ed849e8a5f85b, entries=150, sequenceid=115, filesize=30.2 K 2024-11-23T14:26:48,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/B/dd50874d686941e9972d021dc9a869d6 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/dd50874d686941e9972d021dc9a869d6 2024-11-23T14:26:48,472 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/dd50874d686941e9972d021dc9a869d6, entries=150, sequenceid=115, filesize=11.7 K 2024-11-23T14:26:48,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/C/73d8814609584989a07281c06edc12f1 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/73d8814609584989a07281c06edc12f1 2024-11-23T14:26:48,478 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/73d8814609584989a07281c06edc12f1, entries=150, sequenceid=115, filesize=11.7 K 2024-11-23T14:26:48,478 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:48,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32838 deadline: 1732372068470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:48,478 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:48,478 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:48,478 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:48,480 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for fa04d9e7f6ce6ae3964ded9c7627848b in 1424ms, sequenceid=115, compaction requested=false 2024-11-23T14:26:48,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for fa04d9e7f6ce6ae3964ded9c7627848b: 2024-11-23T14:26:48,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:48,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-23T14:26:48,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-23T14:26:48,483 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-23T14:26:48,483 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3670 sec 2024-11-23T14:26:48,485 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 2.3720 sec 2024-11-23T14:26:48,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on fa04d9e7f6ce6ae3964ded9c7627848b 2024-11-23T14:26:48,487 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T14:26:48,487 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fa04d9e7f6ce6ae3964ded9c7627848b 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-23T14:26:48,495 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112375ff32815055409a9657ef170ff4a07d_fa04d9e7f6ce6ae3964ded9c7627848b is 50, key is test_row_0/A:col10/1732372007321/Put/seqid=0 2024-11-23T14:26:48,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742219_1395 (size=14794) 2024-11-23T14:26:48,508 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:26:48,513 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112375ff32815055409a9657ef170ff4a07d_fa04d9e7f6ce6ae3964ded9c7627848b to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112375ff32815055409a9657ef170ff4a07d_fa04d9e7f6ce6ae3964ded9c7627848b 2024-11-23T14:26:48,519 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/A/7539e5610b0c455a8ae787d6fa8c2cfe, store: [table=TestAcidGuarantees family=A region=fa04d9e7f6ce6ae3964ded9c7627848b] 2024-11-23T14:26:48,520 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/A/7539e5610b0c455a8ae787d6fa8c2cfe is 175, key is test_row_0/A:col10/1732372007321/Put/seqid=0 2024-11-23T14:26:48,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742220_1396 (size=39749) 2024-11-23T14:26:48,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:48,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32848 deadline: 1732372068533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:48,543 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:48,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:48,543 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:48,543 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:48,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32856 deadline: 1732372068533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:48,543 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:48,543 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:48,543 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:48,549 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:48,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32896 deadline: 1732372068539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:48,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:48,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:48,552 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:48,552 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:48,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32882 deadline: 1732372068543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:48,552 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:48,552 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:48,552 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:48,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:48,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32856 deadline: 1732372068650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:48,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:48,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32848 deadline: 1732372068650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:48,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:48,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:48,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:48,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:48,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:48,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:48,667 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:48,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32882 deadline: 1732372068660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:48,668 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:48,668 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:48,668 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:48,668 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:48,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32896 deadline: 1732372068660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:48,668 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:48,668 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:48,668 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:48,867 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:48,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32856 deadline: 1732372068860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:48,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:48,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32848 deadline: 1732372068860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:48,868 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:48,868 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:48,868 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:48,870 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:48,870 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:48,870 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:48,880 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:48,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32882 deadline: 1732372068870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:48,881 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:48,881 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:48,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32896 deadline: 1732372068870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:48,881 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:48,881 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:48,881 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:48,881 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:48,881 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:48,931 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=134, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/A/7539e5610b0c455a8ae787d6fa8c2cfe 2024-11-23T14:26:48,957 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/B/3f5e5640679f40d798e5463bf964eaba is 50, key is test_row_0/B:col10/1732372007321/Put/seqid=0 2024-11-23T14:26:48,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742221_1397 (size=12151) 2024-11-23T14:26:48,984 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/B/3f5e5640679f40d798e5463bf964eaba 2024-11-23T14:26:48,992 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/C/7d20fb2343a14c5c89e32337948bc156 is 50, key is test_row_0/C:col10/1732372007321/Put/seqid=0 2024-11-23T14:26:48,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742222_1398 (size=12151) 2024-11-23T14:26:49,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:49,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32848 deadline: 1732372069180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:49,185 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:49,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:49,185 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:49,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32856 deadline: 1732372069180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:49,185 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:49,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:49,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:49,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:49,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:49,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32882 deadline: 1732372069190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:49,195 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:49,195 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:49,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:49,195 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:49,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32896 deadline: 1732372069190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:49,195 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:49,195 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:49,195 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:49,398 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/C/7d20fb2343a14c5c89e32337948bc156 2024-11-23T14:26:49,401 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/A/7539e5610b0c455a8ae787d6fa8c2cfe as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/7539e5610b0c455a8ae787d6fa8c2cfe 2024-11-23T14:26:49,404 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/7539e5610b0c455a8ae787d6fa8c2cfe, entries=200, sequenceid=134, filesize=38.8 K 2024-11-23T14:26:49,405 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/B/3f5e5640679f40d798e5463bf964eaba as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/3f5e5640679f40d798e5463bf964eaba 2024-11-23T14:26:49,409 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/3f5e5640679f40d798e5463bf964eaba, entries=150, sequenceid=134, filesize=11.9 K 2024-11-23T14:26:49,410 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/C/7d20fb2343a14c5c89e32337948bc156 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/7d20fb2343a14c5c89e32337948bc156 2024-11-23T14:26:49,413 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/7d20fb2343a14c5c89e32337948bc156, entries=150, sequenceid=134, filesize=11.9 K 2024-11-23T14:26:49,414 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for fa04d9e7f6ce6ae3964ded9c7627848b in 927ms, sequenceid=134, compaction requested=true 2024-11-23T14:26:49,414 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fa04d9e7f6ce6ae3964ded9c7627848b: 2024-11-23T14:26:49,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fa04d9e7f6ce6ae3964ded9c7627848b:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T14:26:49,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:26:49,414 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:26:49,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fa04d9e7f6ce6ae3964ded9c7627848b:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T14:26:49,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:26:49,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fa04d9e7f6ce6ae3964ded9c7627848b:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T14:26:49,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-23T14:26:49,414 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:26:49,415 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:26:49,415 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:26:49,415 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1541): fa04d9e7f6ce6ae3964ded9c7627848b/A is initiating minor compaction (all files) 2024-11-23T14:26:49,415 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1541): fa04d9e7f6ce6ae3964ded9c7627848b/B is initiating minor compaction (all files) 2024-11-23T14:26:49,415 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fa04d9e7f6ce6ae3964ded9c7627848b/A in TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:49,415 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fa04d9e7f6ce6ae3964ded9c7627848b/B in TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:49,415 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/615a4c27ae5d411987c99d2ca1199948, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/bff842b8b05b4f7eb17ed849e8a5f85b, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/7539e5610b0c455a8ae787d6fa8c2cfe] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp, totalSize=99.5 K 2024-11-23T14:26:49,415 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/419ea993d355482ab74361efef7ae7ac, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/dd50874d686941e9972d021dc9a869d6, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/3f5e5640679f40d798e5463bf964eaba] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp, totalSize=35.5 K 2024-11-23T14:26:49,415 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:49,415 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. files: [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/615a4c27ae5d411987c99d2ca1199948, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/bff842b8b05b4f7eb17ed849e8a5f85b, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/7539e5610b0c455a8ae787d6fa8c2cfe] 2024-11-23T14:26:49,416 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 615a4c27ae5d411987c99d2ca1199948, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732372005417 2024-11-23T14:26:49,416 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 419ea993d355482ab74361efef7ae7ac, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732372005417 2024-11-23T14:26:49,416 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting bff842b8b05b4f7eb17ed849e8a5f85b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732372006115 2024-11-23T14:26:49,416 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting dd50874d686941e9972d021dc9a869d6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732372006115 2024-11-23T14:26:49,416 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7539e5610b0c455a8ae787d6fa8c2cfe, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732372007321 2024-11-23T14:26:49,416 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f5e5640679f40d798e5463bf964eaba, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732372007321 2024-11-23T14:26:49,428 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fa04d9e7f6ce6ae3964ded9c7627848b] 2024-11-23T14:26:49,431 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241123f2125d1a607946619b6032993852efdb_fa04d9e7f6ce6ae3964ded9c7627848b store=[table=TestAcidGuarantees family=A region=fa04d9e7f6ce6ae3964ded9c7627848b] 2024-11-23T14:26:49,431 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fa04d9e7f6ce6ae3964ded9c7627848b#B#compaction#343 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:26:49,431 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/B/37a732cae24f460797368f4d16a5ea11 is 50, key is test_row_0/B:col10/1732372007321/Put/seqid=0 2024-11-23T14:26:49,432 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241123f2125d1a607946619b6032993852efdb_fa04d9e7f6ce6ae3964ded9c7627848b, store=[table=TestAcidGuarantees family=A region=fa04d9e7f6ce6ae3964ded9c7627848b] 2024-11-23T14:26:49,433 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123f2125d1a607946619b6032993852efdb_fa04d9e7f6ce6ae3964ded9c7627848b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fa04d9e7f6ce6ae3964ded9c7627848b] 2024-11-23T14:26:49,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742224_1400 (size=4469) 2024-11-23T14:26:49,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742223_1399 (size=12459) 2024-11-23T14:26:49,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on fa04d9e7f6ce6ae3964ded9c7627848b 2024-11-23T14:26:49,495 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T14:26:49,495 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fa04d9e7f6ce6ae3964ded9c7627848b 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-23T14:26:49,501 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411231a0d67a3b21b46bfa0200ede5a5c15bd_fa04d9e7f6ce6ae3964ded9c7627848b is 50, key is test_row_0/A:col10/1732372009494/Put/seqid=0 2024-11-23T14:26:49,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742225_1401 (size=14794) 2024-11-23T14:26:49,506 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:26:49,508 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411231a0d67a3b21b46bfa0200ede5a5c15bd_fa04d9e7f6ce6ae3964ded9c7627848b to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411231a0d67a3b21b46bfa0200ede5a5c15bd_fa04d9e7f6ce6ae3964ded9c7627848b 2024-11-23T14:26:49,509 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/A/ada406989bf94772a9b603e22f4d1d45, store: [table=TestAcidGuarantees family=A region=fa04d9e7f6ce6ae3964ded9c7627848b] 2024-11-23T14:26:49,509 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/A/ada406989bf94772a9b603e22f4d1d45 is 175, key is test_row_0/A:col10/1732372009494/Put/seqid=0 2024-11-23T14:26:49,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742226_1402 (size=39749) 2024-11-23T14:26:49,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:49,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32838 deadline: 1732372069586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:49,591 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:49,591 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:49,591 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:49,693 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:49,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32848 deadline: 1732372069690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:49,694 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:49,694 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:49,694 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:49,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32856 deadline: 1732372069690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:49,694 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:49,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:49,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:49,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:49,704 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:49,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32882 deadline: 1732372069699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:49,704 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:49,704 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:49,704 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:49,704 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:49,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32896 deadline: 1732372069700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:49,705 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:49,705 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:49,705 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:49,705 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:49,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32838 deadline: 1732372069700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:49,705 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:49,705 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:49,705 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:49,848 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fa04d9e7f6ce6ae3964ded9c7627848b#A#compaction#342 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:26:49,849 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/A/6449f3cc1b6a48d68fb958cf1fc864ba is 175, key is test_row_0/A:col10/1732372007321/Put/seqid=0 2024-11-23T14:26:49,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742227_1403 (size=31413) 2024-11-23T14:26:49,854 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/B/37a732cae24f460797368f4d16a5ea11 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/37a732cae24f460797368f4d16a5ea11 2024-11-23T14:26:49,858 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/A/6449f3cc1b6a48d68fb958cf1fc864ba as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/6449f3cc1b6a48d68fb958cf1fc864ba 2024-11-23T14:26:49,860 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fa04d9e7f6ce6ae3964ded9c7627848b/B of fa04d9e7f6ce6ae3964ded9c7627848b into 37a732cae24f460797368f4d16a5ea11(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:26:49,860 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fa04d9e7f6ce6ae3964ded9c7627848b: 2024-11-23T14:26:49,860 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., storeName=fa04d9e7f6ce6ae3964ded9c7627848b/B, priority=13, startTime=1732372009414; duration=0sec 2024-11-23T14:26:49,860 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:26:49,860 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fa04d9e7f6ce6ae3964ded9c7627848b:B 2024-11-23T14:26:49,860 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:26:49,861 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:26:49,861 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1541): fa04d9e7f6ce6ae3964ded9c7627848b/C is initiating minor compaction (all files) 2024-11-23T14:26:49,861 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fa04d9e7f6ce6ae3964ded9c7627848b/C in TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:49,861 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/c3a02fdf495c45c19d9d5ba2b579e842, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/73d8814609584989a07281c06edc12f1, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/7d20fb2343a14c5c89e32337948bc156] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp, totalSize=35.5 K 2024-11-23T14:26:49,861 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fa04d9e7f6ce6ae3964ded9c7627848b/A of fa04d9e7f6ce6ae3964ded9c7627848b into 6449f3cc1b6a48d68fb958cf1fc864ba(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:26:49,861 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting c3a02fdf495c45c19d9d5ba2b579e842, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732372005417 2024-11-23T14:26:49,861 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fa04d9e7f6ce6ae3964ded9c7627848b: 2024-11-23T14:26:49,861 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., storeName=fa04d9e7f6ce6ae3964ded9c7627848b/A, priority=13, startTime=1732372009414; duration=0sec 2024-11-23T14:26:49,862 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:26:49,862 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 73d8814609584989a07281c06edc12f1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732372006115 2024-11-23T14:26:49,862 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fa04d9e7f6ce6ae3964ded9c7627848b:A 2024-11-23T14:26:49,862 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d20fb2343a14c5c89e32337948bc156, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732372007321 2024-11-23T14:26:49,872 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fa04d9e7f6ce6ae3964ded9c7627848b#C#compaction#345 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:26:49,873 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/C/41abb2539dbb41e1a7253f096311648b is 50, key is test_row_0/C:col10/1732372007321/Put/seqid=0 2024-11-23T14:26:49,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742228_1404 (size=12459) 2024-11-23T14:26:49,914 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=155, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/A/ada406989bf94772a9b603e22f4d1d45 2024-11-23T14:26:49,916 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:49,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32838 deadline: 1732372069910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:49,916 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:49,916 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:49,917 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:49,921 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/B/17913e69c8054e69830264c1508cf353 is 50, key is test_row_0/B:col10/1732372009494/Put/seqid=0 2024-11-23T14:26:49,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742229_1405 (size=12151) 2024-11-23T14:26:50,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:50,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32838 deadline: 1732372070220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:50,228 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:50,228 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:50,228 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:50,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-23T14:26:50,272 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T14:26:50,272 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-23T14:26:50,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-23T14:26:50,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-23T14:26:50,274 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T14:26:50,275 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T14:26:50,275 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T14:26:50,291 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/C/41abb2539dbb41e1a7253f096311648b as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/41abb2539dbb41e1a7253f096311648b 2024-11-23T14:26:50,296 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fa04d9e7f6ce6ae3964ded9c7627848b/C of fa04d9e7f6ce6ae3964ded9c7627848b into 41abb2539dbb41e1a7253f096311648b(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:26:50,296 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fa04d9e7f6ce6ae3964ded9c7627848b: 2024-11-23T14:26:50,296 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., storeName=fa04d9e7f6ce6ae3964ded9c7627848b/C, priority=13, startTime=1732372009414; duration=0sec 2024-11-23T14:26:50,296 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:26:50,296 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fa04d9e7f6ce6ae3964ded9c7627848b:C 2024-11-23T14:26:50,326 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/B/17913e69c8054e69830264c1508cf353 2024-11-23T14:26:50,346 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/C/33704be76b6e4bd996078a7193069229 is 50, key is test_row_0/C:col10/1732372009494/Put/seqid=0 2024-11-23T14:26:50,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-23T14:26:50,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742230_1406 (size=12151) 2024-11-23T14:26:50,390 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/C/33704be76b6e4bd996078a7193069229 2024-11-23T14:26:50,402 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/A/ada406989bf94772a9b603e22f4d1d45 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/ada406989bf94772a9b603e22f4d1d45 2024-11-23T14:26:50,410 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/ada406989bf94772a9b603e22f4d1d45, entries=200, sequenceid=155, filesize=38.8 K 2024-11-23T14:26:50,411 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/B/17913e69c8054e69830264c1508cf353 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/17913e69c8054e69830264c1508cf353 2024-11-23T14:26:50,422 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/17913e69c8054e69830264c1508cf353, entries=150, sequenceid=155, filesize=11.9 K 2024-11-23T14:26:50,424 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/C/33704be76b6e4bd996078a7193069229 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/33704be76b6e4bd996078a7193069229 2024-11-23T14:26:50,427 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-23T14:26:50,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:50,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. as already flushing 2024-11-23T14:26:50,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:50,428 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:50,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:50,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:50,441 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/33704be76b6e4bd996078a7193069229, entries=150, sequenceid=155, filesize=11.9 K 2024-11-23T14:26:50,442 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for fa04d9e7f6ce6ae3964ded9c7627848b in 947ms, sequenceid=155, compaction requested=false 2024-11-23T14:26:50,442 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fa04d9e7f6ce6ae3964ded9c7627848b: 2024-11-23T14:26:50,582 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-23T14:26:50,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:50,582 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing fa04d9e7f6ce6ae3964ded9c7627848b 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-23T14:26:50,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-23T14:26:50,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123b202aba90e64400183d3577ad6e8921b_fa04d9e7f6ce6ae3964ded9c7627848b is 50, key is test_row_0/A:col10/1732372009577/Put/seqid=0 2024-11-23T14:26:50,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742231_1407 (size=12304) 2024-11-23T14:26:50,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on fa04d9e7f6ce6ae3964ded9c7627848b 2024-11-23T14:26:50,704 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. as already flushing 2024-11-23T14:26:50,755 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:50,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32856 deadline: 1732372070745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:50,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:50,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:50,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:50,756 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:50,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32882 deadline: 1732372070748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:50,756 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:50,756 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:50,756 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:50,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:50,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32838 deadline: 1732372070749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:50,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:50,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32896 deadline: 1732372070754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:50,759 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:50,759 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:50,759 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:50,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:50,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:50,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:50,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:50,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32848 deadline: 1732372070755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:50,766 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:50,766 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:50,766 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:50,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:50,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32856 deadline: 1732372070860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:50,868 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:50,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32882 deadline: 1732372070860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:50,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:50,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:50,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:50,868 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:50,868 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:50,868 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:50,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:50,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32896 deadline: 1732372070870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:50,875 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:50,875 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:50,875 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:50,877 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:50,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32838 deadline: 1732372070871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:50,877 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:50,877 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:50,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32848 deadline: 1732372070871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:50,877 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:50,877 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:50,878 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:50,878 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:50,878 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:50,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-23T14:26:51,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:26:51,028 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123b202aba90e64400183d3577ad6e8921b_fa04d9e7f6ce6ae3964ded9c7627848b to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123b202aba90e64400183d3577ad6e8921b_fa04d9e7f6ce6ae3964ded9c7627848b 2024-11-23T14:26:51,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/A/ef53055587144cea85be65ae94d248e4, store: [table=TestAcidGuarantees family=A region=fa04d9e7f6ce6ae3964ded9c7627848b] 2024-11-23T14:26:51,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/A/ef53055587144cea85be65ae94d248e4 is 175, key is test_row_0/A:col10/1732372009577/Put/seqid=0 2024-11-23T14:26:51,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742232_1408 (size=31105) 2024-11-23T14:26:51,032 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=173, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/A/ef53055587144cea85be65ae94d248e4 2024-11-23T14:26:51,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/B/e9d3253b129d425ea646be0fdfd9de58 is 50, key is test_row_0/B:col10/1732372009577/Put/seqid=0 2024-11-23T14:26:51,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742233_1409 (size=12151) 2024-11-23T14:26:51,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:51,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32856 deadline: 1732372071070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:51,077 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:51,077 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:51,077 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:51,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:51,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32882 deadline: 1732372071080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:51,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:51,086 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:51,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32896 deadline: 1732372071080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:51,086 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:51,087 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:51,087 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:51,087 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:51,087 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:51,095 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:51,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32838 deadline: 1732372071090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:51,096 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:51,096 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:51,096 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:51,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32848 deadline: 1732372071090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:51,096 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:51,096 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:51,096 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:51,096 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:51,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:51,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32856 deadline: 1732372071380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:51,388 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:51,388 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:51,388 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:51,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:51,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32882 deadline: 1732372071390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:51,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:51,399 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:51,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32896 deadline: 1732372071390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:51,399 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:51,399 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:51,400 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:51,400 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:51,400 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:51,400 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:51,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32838 deadline: 1732372071400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:51,400 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:51,400 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:51,400 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:51,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-23T14:26:51,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:51,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32848 deadline: 1732372071410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:51,416 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:51,416 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:51,416 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:51,443 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/B/e9d3253b129d425ea646be0fdfd9de58 2024-11-23T14:26:51,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/C/c77a33b7c2a04ff9b318b0dab8c424aa is 50, key is test_row_0/C:col10/1732372009577/Put/seqid=0 2024-11-23T14:26:51,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742234_1410 (size=12151) 2024-11-23T14:26:51,856 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/C/c77a33b7c2a04ff9b318b0dab8c424aa 2024-11-23T14:26:51,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/A/ef53055587144cea85be65ae94d248e4 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/ef53055587144cea85be65ae94d248e4 2024-11-23T14:26:51,866 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/ef53055587144cea85be65ae94d248e4, entries=150, sequenceid=173, filesize=30.4 K 2024-11-23T14:26:51,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/B/e9d3253b129d425ea646be0fdfd9de58 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/e9d3253b129d425ea646be0fdfd9de58 2024-11-23T14:26:51,872 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/e9d3253b129d425ea646be0fdfd9de58, entries=150, sequenceid=173, filesize=11.9 K 2024-11-23T14:26:51,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/C/c77a33b7c2a04ff9b318b0dab8c424aa as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/c77a33b7c2a04ff9b318b0dab8c424aa 2024-11-23T14:26:51,879 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/c77a33b7c2a04ff9b318b0dab8c424aa, entries=150, sequenceid=173, filesize=11.9 K 2024-11-23T14:26:51,880 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for fa04d9e7f6ce6ae3964ded9c7627848b in 1298ms, sequenceid=173, compaction requested=true 2024-11-23T14:26:51,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for fa04d9e7f6ce6ae3964ded9c7627848b: 2024-11-23T14:26:51,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:51,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-23T14:26:51,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-23T14:26:51,884 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-23T14:26:51,884 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6070 sec 2024-11-23T14:26:51,889 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 1.6160 sec 2024-11-23T14:26:51,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on fa04d9e7f6ce6ae3964ded9c7627848b 2024-11-23T14:26:51,896 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T14:26:51,897 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fa04d9e7f6ce6ae3964ded9c7627848b 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-23T14:26:51,905 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411237f58e5f04d3545b9b1e5a943158cb2d5_fa04d9e7f6ce6ae3964ded9c7627848b is 50, key is test_row_0/A:col10/1732372010748/Put/seqid=0 2024-11-23T14:26:51,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742235_1411 (size=14794) 2024-11-23T14:26:51,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:51,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:51,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32896 deadline: 1732372071933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:51,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32838 deadline: 1732372071935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:51,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:51,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:51,949 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:51,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32856 deadline: 1732372071935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:51,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32882 deadline: 1732372071936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:51,949 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:51,949 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:51,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:51,949 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:51,949 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:51,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32848 deadline: 1732372071936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:51,949 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:51,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:51,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:51,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:51,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:51,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:51,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:51,951 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:51,951 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:51,951 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:52,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:52,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32838 deadline: 1732372072060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:52,062 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:52,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:52,062 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:52,062 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:52,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32896 deadline: 1732372072060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:52,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:52,062 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:52,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32848 deadline: 1732372072060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:52,062 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:52,062 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:52,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:52,063 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:52,063 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:52,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32856 deadline: 1732372072060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:52,063 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:52,063 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:52,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:52,063 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:52,063 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:52,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32882 deadline: 1732372072060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:52,063 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:52,063 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:52,063 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:52,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:52,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32838 deadline: 1732372072270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:52,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:52,271 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:52,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32896 deadline: 1732372072270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:52,271 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:52,271 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:52,272 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:52,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:52,272 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:52,272 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:52,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32848 deadline: 1732372072270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:52,272 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:52,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:52,272 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:52,272 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:52,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32856 deadline: 1732372072270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:52,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:52,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:52,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:52,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32882 deadline: 1732372072270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:52,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:52,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:52,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:52,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:52,314 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:26:52,317 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411237f58e5f04d3545b9b1e5a943158cb2d5_fa04d9e7f6ce6ae3964ded9c7627848b to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411237f58e5f04d3545b9b1e5a943158cb2d5_fa04d9e7f6ce6ae3964ded9c7627848b 2024-11-23T14:26:52,318 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/A/51e49563a8584e148b64a7932bd1312f, store: [table=TestAcidGuarantees family=A region=fa04d9e7f6ce6ae3964ded9c7627848b] 2024-11-23T14:26:52,319 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/A/51e49563a8584e148b64a7932bd1312f is 175, key is test_row_0/A:col10/1732372010748/Put/seqid=0 2024-11-23T14:26:52,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742236_1412 (size=39749) 2024-11-23T14:26:52,326 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=195, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/A/51e49563a8584e148b64a7932bd1312f 2024-11-23T14:26:52,332 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/B/8baa731b18d84c399d5f2a96027c0781 is 50, key is test_row_0/B:col10/1732372010748/Put/seqid=0 2024-11-23T14:26:52,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742237_1413 (size=12151) 2024-11-23T14:26:52,341 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/B/8baa731b18d84c399d5f2a96027c0781 2024-11-23T14:26:52,348 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/C/e1322d26948c47d288eca4b8e1bd4edc is 50, key is test_row_0/C:col10/1732372010748/Put/seqid=0 2024-11-23T14:26:52,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742238_1414 (size=12151) 2024-11-23T14:26:52,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-23T14:26:52,421 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T14:26:52,421 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-23T14:26:52,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-23T14:26:52,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-23T14:26:52,423 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T14:26:52,423 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T14:26:52,423 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T14:26:52,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-23T14:26:52,575 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-23T14:26:52,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:52,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. as already flushing 2024-11-23T14:26:52,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:52,576 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:52,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:52,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:52,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:52,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32838 deadline: 1732372072580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:52,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:52,586 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:52,586 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:52,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32896 deadline: 1732372072580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:52,586 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:52,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:52,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:52,587 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:52,587 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:52,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32848 deadline: 1732372072580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:52,587 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:52,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:52,587 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:52,587 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:52,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32856 deadline: 1732372072580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:52,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:52,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:52,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:52,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:52,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32882 deadline: 1732372072580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:52,587 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:52,587 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:52,587 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:52,731 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-23T14:26:52,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:52,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. as already flushing 2024-11-23T14:26:52,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:52,731 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:52,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:52,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:52,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-23T14:26:52,755 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/C/e1322d26948c47d288eca4b8e1bd4edc 2024-11-23T14:26:52,764 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/A/51e49563a8584e148b64a7932bd1312f as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/51e49563a8584e148b64a7932bd1312f 2024-11-23T14:26:52,771 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/51e49563a8584e148b64a7932bd1312f, entries=200, sequenceid=195, filesize=38.8 K 2024-11-23T14:26:52,774 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/B/8baa731b18d84c399d5f2a96027c0781 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/8baa731b18d84c399d5f2a96027c0781 2024-11-23T14:26:52,778 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/8baa731b18d84c399d5f2a96027c0781, entries=150, sequenceid=195, filesize=11.9 K 2024-11-23T14:26:52,779 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/C/e1322d26948c47d288eca4b8e1bd4edc as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/e1322d26948c47d288eca4b8e1bd4edc 2024-11-23T14:26:52,794 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/e1322d26948c47d288eca4b8e1bd4edc, entries=150, sequenceid=195, filesize=11.9 K 2024-11-23T14:26:52,795 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for fa04d9e7f6ce6ae3964ded9c7627848b in 899ms, sequenceid=195, compaction requested=true 2024-11-23T14:26:52,796 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fa04d9e7f6ce6ae3964ded9c7627848b: 2024-11-23T14:26:52,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fa04d9e7f6ce6ae3964ded9c7627848b:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T14:26:52,796 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T14:26:52,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:26:52,796 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T14:26:52,797 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142016 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T14:26:52,797 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1541): fa04d9e7f6ce6ae3964ded9c7627848b/A is initiating minor compaction (all files) 2024-11-23T14:26:52,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fa04d9e7f6ce6ae3964ded9c7627848b:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T14:26:52,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:26:52,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fa04d9e7f6ce6ae3964ded9c7627848b:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T14:26:52,797 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fa04d9e7f6ce6ae3964ded9c7627848b/A in TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:52,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:26:52,797 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/6449f3cc1b6a48d68fb958cf1fc864ba, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/ada406989bf94772a9b603e22f4d1d45, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/ef53055587144cea85be65ae94d248e4, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/51e49563a8584e148b64a7932bd1312f] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp, totalSize=138.7 K 2024-11-23T14:26:52,797 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:52,797 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. files: [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/6449f3cc1b6a48d68fb958cf1fc864ba, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/ada406989bf94772a9b603e22f4d1d45, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/ef53055587144cea85be65ae94d248e4, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/51e49563a8584e148b64a7932bd1312f] 2024-11-23T14:26:52,798 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48912 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T14:26:52,798 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1541): fa04d9e7f6ce6ae3964ded9c7627848b/B is initiating minor compaction (all files) 2024-11-23T14:26:52,798 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fa04d9e7f6ce6ae3964ded9c7627848b/B in TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:52,798 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/37a732cae24f460797368f4d16a5ea11, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/17913e69c8054e69830264c1508cf353, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/e9d3253b129d425ea646be0fdfd9de58, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/8baa731b18d84c399d5f2a96027c0781] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp, totalSize=47.8 K 2024-11-23T14:26:52,798 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6449f3cc1b6a48d68fb958cf1fc864ba, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732372007321 2024-11-23T14:26:52,798 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 37a732cae24f460797368f4d16a5ea11, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732372007321 2024-11-23T14:26:52,798 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 17913e69c8054e69830264c1508cf353, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732372008514 2024-11-23T14:26:52,798 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting ada406989bf94772a9b603e22f4d1d45, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732372008514 2024-11-23T14:26:52,799 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting e9d3253b129d425ea646be0fdfd9de58, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1732372009557 2024-11-23T14:26:52,799 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef53055587144cea85be65ae94d248e4, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1732372009557 2024-11-23T14:26:52,799 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 8baa731b18d84c399d5f2a96027c0781, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732372010748 2024-11-23T14:26:52,799 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 51e49563a8584e148b64a7932bd1312f, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732372010748 2024-11-23T14:26:52,809 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fa04d9e7f6ce6ae3964ded9c7627848b] 2024-11-23T14:26:52,809 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fa04d9e7f6ce6ae3964ded9c7627848b#B#compaction#354 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:26:52,810 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/B/203737f140274d7fa679510220914d05 is 50, key is test_row_0/B:col10/1732372010748/Put/seqid=0 2024-11-23T14:26:52,822 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411238d2ef7282fb0492bac9cd355f59cc2e1_fa04d9e7f6ce6ae3964ded9c7627848b store=[table=TestAcidGuarantees family=A region=fa04d9e7f6ce6ae3964ded9c7627848b] 2024-11-23T14:26:52,824 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411238d2ef7282fb0492bac9cd355f59cc2e1_fa04d9e7f6ce6ae3964ded9c7627848b, store=[table=TestAcidGuarantees family=A region=fa04d9e7f6ce6ae3964ded9c7627848b] 2024-11-23T14:26:52,825 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411238d2ef7282fb0492bac9cd355f59cc2e1_fa04d9e7f6ce6ae3964ded9c7627848b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fa04d9e7f6ce6ae3964ded9c7627848b] 2024-11-23T14:26:52,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742239_1415 (size=12595) 2024-11-23T14:26:52,846 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/B/203737f140274d7fa679510220914d05 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/203737f140274d7fa679510220914d05 2024-11-23T14:26:52,850 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fa04d9e7f6ce6ae3964ded9c7627848b/B of fa04d9e7f6ce6ae3964ded9c7627848b into 203737f140274d7fa679510220914d05(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:26:52,850 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fa04d9e7f6ce6ae3964ded9c7627848b: 2024-11-23T14:26:52,850 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., storeName=fa04d9e7f6ce6ae3964ded9c7627848b/B, priority=12, startTime=1732372012796; duration=0sec 2024-11-23T14:26:52,850 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:26:52,850 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fa04d9e7f6ce6ae3964ded9c7627848b:B 2024-11-23T14:26:52,851 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T14:26:52,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742240_1416 (size=4469) 2024-11-23T14:26:52,854 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48912 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T14:26:52,854 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1541): fa04d9e7f6ce6ae3964ded9c7627848b/C is initiating minor compaction (all files) 2024-11-23T14:26:52,855 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fa04d9e7f6ce6ae3964ded9c7627848b/C in TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:52,855 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/41abb2539dbb41e1a7253f096311648b, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/33704be76b6e4bd996078a7193069229, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/c77a33b7c2a04ff9b318b0dab8c424aa, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/e1322d26948c47d288eca4b8e1bd4edc] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp, totalSize=47.8 K 2024-11-23T14:26:52,855 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 41abb2539dbb41e1a7253f096311648b, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732372007321 2024-11-23T14:26:52,856 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 33704be76b6e4bd996078a7193069229, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732372008514 2024-11-23T14:26:52,856 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting c77a33b7c2a04ff9b318b0dab8c424aa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1732372009557 2024-11-23T14:26:52,857 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting e1322d26948c47d288eca4b8e1bd4edc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732372010748 2024-11-23T14:26:52,864 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fa04d9e7f6ce6ae3964ded9c7627848b#A#compaction#355 average throughput is 0.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:26:52,864 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/A/391b52b702c646c1bee649b6411bc1f8 is 175, key is test_row_0/A:col10/1732372010748/Put/seqid=0 2024-11-23T14:26:52,885 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-23T14:26:52,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:52,885 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing fa04d9e7f6ce6ae3964ded9c7627848b 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-23T14:26:52,892 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fa04d9e7f6ce6ae3964ded9c7627848b#C#compaction#356 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:26:52,892 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/C/44cd185037fa4277a4a53d56e1672f3d is 50, key is test_row_0/C:col10/1732372010748/Put/seqid=0 2024-11-23T14:26:52,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742241_1417 (size=31549) 2024-11-23T14:26:52,920 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/A/391b52b702c646c1bee649b6411bc1f8 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/391b52b702c646c1bee649b6411bc1f8 2024-11-23T14:26:52,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123a6a24b1f38344df09c222fdb0a16acbf_fa04d9e7f6ce6ae3964ded9c7627848b is 50, key is test_row_0/A:col10/1732372011931/Put/seqid=0 2024-11-23T14:26:52,930 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fa04d9e7f6ce6ae3964ded9c7627848b/A of fa04d9e7f6ce6ae3964ded9c7627848b into 391b52b702c646c1bee649b6411bc1f8(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:26:52,930 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fa04d9e7f6ce6ae3964ded9c7627848b: 2024-11-23T14:26:52,930 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., storeName=fa04d9e7f6ce6ae3964ded9c7627848b/A, priority=12, startTime=1732372012796; duration=0sec 2024-11-23T14:26:52,930 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:26:52,930 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fa04d9e7f6ce6ae3964ded9c7627848b:A 2024-11-23T14:26:52,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742242_1418 (size=12595) 2024-11-23T14:26:52,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742243_1419 (size=12304) 2024-11-23T14:26:52,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:26:52,981 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123a6a24b1f38344df09c222fdb0a16acbf_fa04d9e7f6ce6ae3964ded9c7627848b to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123a6a24b1f38344df09c222fdb0a16acbf_fa04d9e7f6ce6ae3964ded9c7627848b 2024-11-23T14:26:52,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/A/848550e06ab04428862d25b3b5cdbebe, store: [table=TestAcidGuarantees family=A region=fa04d9e7f6ce6ae3964ded9c7627848b] 2024-11-23T14:26:52,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/A/848550e06ab04428862d25b3b5cdbebe is 175, key is test_row_0/A:col10/1732372011931/Put/seqid=0 2024-11-23T14:26:53,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742244_1420 (size=31105) 2024-11-23T14:26:53,024 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=210, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/A/848550e06ab04428862d25b3b5cdbebe 2024-11-23T14:26:53,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/B/013d6487fbec401393b216c551c18d84 is 50, key is test_row_0/B:col10/1732372011931/Put/seqid=0 2024-11-23T14:26:53,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-23T14:26:53,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742245_1421 (size=12151) 2024-11-23T14:26:53,053 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/B/013d6487fbec401393b216c551c18d84 2024-11-23T14:26:53,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/C/d4e974bb019e4ef8a87d3b758859d8a9 is 50, key is test_row_0/C:col10/1732372011931/Put/seqid=0 2024-11-23T14:26:53,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742246_1422 (size=12151) 2024-11-23T14:26:53,095 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/C/d4e974bb019e4ef8a87d3b758859d8a9 2024-11-23T14:26:53,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on fa04d9e7f6ce6ae3964ded9c7627848b 2024-11-23T14:26:53,096 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. as already flushing 2024-11-23T14:26:53,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/A/848550e06ab04428862d25b3b5cdbebe as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/848550e06ab04428862d25b3b5cdbebe 2024-11-23T14:26:53,108 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/848550e06ab04428862d25b3b5cdbebe, entries=150, sequenceid=210, filesize=30.4 K 2024-11-23T14:26:53,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/B/013d6487fbec401393b216c551c18d84 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/013d6487fbec401393b216c551c18d84 2024-11-23T14:26:53,114 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/013d6487fbec401393b216c551c18d84, entries=150, sequenceid=210, filesize=11.9 K 2024-11-23T14:26:53,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/C/d4e974bb019e4ef8a87d3b758859d8a9 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/d4e974bb019e4ef8a87d3b758859d8a9 2024-11-23T14:26:53,120 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/d4e974bb019e4ef8a87d3b758859d8a9, entries=150, sequenceid=210, filesize=11.9 K 2024-11-23T14:26:53,122 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=67.09 KB/68700 for fa04d9e7f6ce6ae3964ded9c7627848b in 237ms, sequenceid=210, compaction requested=false 2024-11-23T14:26:53,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for fa04d9e7f6ce6ae3964ded9c7627848b: 2024-11-23T14:26:53,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:53,122 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T14:26:53,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-23T14:26:53,122 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fa04d9e7f6ce6ae3964ded9c7627848b 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-23T14:26:53,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on fa04d9e7f6ce6ae3964ded9c7627848b 2024-11-23T14:26:53,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-23T14:26:53,133 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-23T14:26:53,133 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 701 msec 2024-11-23T14:26:53,135 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 713 msec 2024-11-23T14:26:53,139 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123cd7e22eb6617466db192730c54e735df_fa04d9e7f6ce6ae3964ded9c7627848b is 50, key is test_row_0/A:col10/1732372013105/Put/seqid=0 2024-11-23T14:26:53,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742247_1423 (size=19774) 2024-11-23T14:26:53,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:53,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32856 deadline: 1732372073178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:53,190 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:53,190 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:53,190 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:53,196 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:53,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32882 deadline: 1732372073179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:53,196 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:53,196 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:53,196 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:53,196 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:53,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32838 deadline: 1732372073181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:53,197 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:53,197 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:53,197 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:53,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:53,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32896 deadline: 1732372073182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:53,197 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:53,197 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:53,197 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:53,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:53,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32848 deadline: 1732372073183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:53,197 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:53,197 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:53,197 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:53,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:53,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32856 deadline: 1732372073300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:53,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:53,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32882 deadline: 1732372073300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:53,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:53,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32838 deadline: 1732372073300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:53,311 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:53,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32848 deadline: 1732372073300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:53,311 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:53,311 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:53,311 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:53,311 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:53,311 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:53,311 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:53,311 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:53,311 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:53,311 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:53,311 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:53,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32896 deadline: 1732372073300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:53,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:53,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:53,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:53,311 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:53,311 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:53,311 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:53,346 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/C/44cd185037fa4277a4a53d56e1672f3d as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/44cd185037fa4277a4a53d56e1672f3d 2024-11-23T14:26:53,364 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fa04d9e7f6ce6ae3964ded9c7627848b/C of fa04d9e7f6ce6ae3964ded9c7627848b into 44cd185037fa4277a4a53d56e1672f3d(size=12.3 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:26:53,364 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fa04d9e7f6ce6ae3964ded9c7627848b: 2024-11-23T14:26:53,364 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., storeName=fa04d9e7f6ce6ae3964ded9c7627848b/C, priority=12, startTime=1732372012797; duration=0sec 2024-11-23T14:26:53,364 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:26:53,364 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fa04d9e7f6ce6ae3964ded9c7627848b:C 2024-11-23T14:26:53,527 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:53,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32838 deadline: 1732372073521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:53,528 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:53,528 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:53,528 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:53,528 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:53,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32882 deadline: 1732372073521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:53,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:53,528 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:53,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:53,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:53,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32848 deadline: 1732372073521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:53,529 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:53,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32856 deadline: 1732372073521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:53,529 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:53,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32896 deadline: 1732372073521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:53,530 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:53,530 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:53,530 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:53,530 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:53,530 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:53,530 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:53,530 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:53,530 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:53,530 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:53,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-23T14:26:53,560 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T14:26:53,561 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-23T14:26:53,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-23T14:26:53,563 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T14:26:53,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-23T14:26:53,563 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:26:53,563 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T14:26:53,564 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T14:26:53,567 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123cd7e22eb6617466db192730c54e735df_fa04d9e7f6ce6ae3964ded9c7627848b to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123cd7e22eb6617466db192730c54e735df_fa04d9e7f6ce6ae3964ded9c7627848b 2024-11-23T14:26:53,569 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/A/aebc6545cc5d4b16af8456cfa25c7272, store: [table=TestAcidGuarantees family=A region=fa04d9e7f6ce6ae3964ded9c7627848b] 2024-11-23T14:26:53,570 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/A/aebc6545cc5d4b16af8456cfa25c7272 is 175, key is test_row_0/A:col10/1732372013105/Put/seqid=0 2024-11-23T14:26:53,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742248_1424 (size=57033) 2024-11-23T14:26:53,599 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=226, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/A/aebc6545cc5d4b16af8456cfa25c7272 2024-11-23T14:26:53,616 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/B/d0910d4aecb843c3b86db8a0ebea9ca2 is 50, key is test_row_0/B:col10/1732372013105/Put/seqid=0 2024-11-23T14:26:53,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742249_1425 (size=12151) 2024-11-23T14:26:53,638 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/B/d0910d4aecb843c3b86db8a0ebea9ca2 2024-11-23T14:26:53,649 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/C/65bb71c263964fdeb0fa0cccdf8b936c is 50, key is test_row_0/C:col10/1732372013105/Put/seqid=0 2024-11-23T14:26:53,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-23T14:26:53,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742250_1426 (size=12151) 2024-11-23T14:26:53,675 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/C/65bb71c263964fdeb0fa0cccdf8b936c 2024-11-23T14:26:53,682 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/A/aebc6545cc5d4b16af8456cfa25c7272 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/aebc6545cc5d4b16af8456cfa25c7272 2024-11-23T14:26:53,686 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/aebc6545cc5d4b16af8456cfa25c7272, entries=300, sequenceid=226, filesize=55.7 K 2024-11-23T14:26:53,687 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/B/d0910d4aecb843c3b86db8a0ebea9ca2 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/d0910d4aecb843c3b86db8a0ebea9ca2 2024-11-23T14:26:53,692 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/d0910d4aecb843c3b86db8a0ebea9ca2, entries=150, sequenceid=226, filesize=11.9 K 2024-11-23T14:26:53,693 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/C/65bb71c263964fdeb0fa0cccdf8b936c as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/65bb71c263964fdeb0fa0cccdf8b936c 2024-11-23T14:26:53,697 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/65bb71c263964fdeb0fa0cccdf8b936c, entries=150, sequenceid=226, filesize=11.9 K 2024-11-23T14:26:53,697 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for fa04d9e7f6ce6ae3964ded9c7627848b in 575ms, sequenceid=226, compaction requested=true 2024-11-23T14:26:53,698 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fa04d9e7f6ce6ae3964ded9c7627848b: 2024-11-23T14:26:53,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fa04d9e7f6ce6ae3964ded9c7627848b:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T14:26:53,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:26:53,698 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:26:53,698 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:26:53,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fa04d9e7f6ce6ae3964ded9c7627848b:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T14:26:53,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:26:53,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fa04d9e7f6ce6ae3964ded9c7627848b:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T14:26:53,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:26:53,699 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:26:53,699 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1541): fa04d9e7f6ce6ae3964ded9c7627848b/B is initiating minor compaction (all files) 2024-11-23T14:26:53,699 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fa04d9e7f6ce6ae3964ded9c7627848b/B in TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:53,699 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/203737f140274d7fa679510220914d05, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/013d6487fbec401393b216c551c18d84, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/d0910d4aecb843c3b86db8a0ebea9ca2] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp, totalSize=36.0 K 2024-11-23T14:26:53,700 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 203737f140274d7fa679510220914d05, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732372010748 2024-11-23T14:26:53,700 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 013d6487fbec401393b216c551c18d84, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1732372011931 2024-11-23T14:26:53,700 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting d0910d4aecb843c3b86db8a0ebea9ca2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1732372013105 2024-11-23T14:26:53,701 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 119687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:26:53,701 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1541): fa04d9e7f6ce6ae3964ded9c7627848b/A is initiating minor compaction (all files) 2024-11-23T14:26:53,702 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fa04d9e7f6ce6ae3964ded9c7627848b/A in TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:53,702 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/391b52b702c646c1bee649b6411bc1f8, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/848550e06ab04428862d25b3b5cdbebe, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/aebc6545cc5d4b16af8456cfa25c7272] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp, totalSize=116.9 K 2024-11-23T14:26:53,702 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:53,702 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. files: [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/391b52b702c646c1bee649b6411bc1f8, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/848550e06ab04428862d25b3b5cdbebe, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/aebc6545cc5d4b16af8456cfa25c7272] 2024-11-23T14:26:53,702 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 391b52b702c646c1bee649b6411bc1f8, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732372010748 2024-11-23T14:26:53,702 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 848550e06ab04428862d25b3b5cdbebe, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1732372011931 2024-11-23T14:26:53,703 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting aebc6545cc5d4b16af8456cfa25c7272, keycount=300, bloomtype=ROW, size=55.7 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1732372013105 2024-11-23T14:26:53,717 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-23T14:26:53,717 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fa04d9e7f6ce6ae3964ded9c7627848b] 2024-11-23T14:26:53,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:53,717 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing fa04d9e7f6ce6ae3964ded9c7627848b 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-23T14:26:53,718 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fa04d9e7f6ce6ae3964ded9c7627848b#B#compaction#363 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:26:53,718 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/B/7337d9279f954897bc434aed573f8aec is 50, key is test_row_0/B:col10/1732372013105/Put/seqid=0 2024-11-23T14:26:53,720 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411233a7426e7bd214ce69b925e228e883af9_fa04d9e7f6ce6ae3964ded9c7627848b store=[table=TestAcidGuarantees family=A region=fa04d9e7f6ce6ae3964ded9c7627848b] 2024-11-23T14:26:53,722 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411233a7426e7bd214ce69b925e228e883af9_fa04d9e7f6ce6ae3964ded9c7627848b, store=[table=TestAcidGuarantees family=A region=fa04d9e7f6ce6ae3964ded9c7627848b] 2024-11-23T14:26:53,722 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411233a7426e7bd214ce69b925e228e883af9_fa04d9e7f6ce6ae3964ded9c7627848b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fa04d9e7f6ce6ae3964ded9c7627848b] 2024-11-23T14:26:53,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411231fc2cab5c016494e812e4d67b016a277_fa04d9e7f6ce6ae3964ded9c7627848b is 50, key is test_row_0/A:col10/1732372013181/Put/seqid=0 2024-11-23T14:26:53,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742251_1427 (size=12697) 2024-11-23T14:26:53,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742252_1428 (size=4469) 2024-11-23T14:26:53,769 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fa04d9e7f6ce6ae3964ded9c7627848b#A#compaction#364 average throughput is 0.47 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:26:53,769 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/A/936fc6eca3424aaeb9dc7d75ec6f0400 is 175, key is test_row_0/A:col10/1732372013105/Put/seqid=0 2024-11-23T14:26:53,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742253_1429 (size=12304) 2024-11-23T14:26:53,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:26:53,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742254_1430 (size=31651) 2024-11-23T14:26:53,775 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411231fc2cab5c016494e812e4d67b016a277_fa04d9e7f6ce6ae3964ded9c7627848b to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411231fc2cab5c016494e812e4d67b016a277_fa04d9e7f6ce6ae3964ded9c7627848b 2024-11-23T14:26:53,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/A/a39f40eee3d449d9983773eb0ac23deb, store: [table=TestAcidGuarantees family=A region=fa04d9e7f6ce6ae3964ded9c7627848b] 2024-11-23T14:26:53,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/A/a39f40eee3d449d9983773eb0ac23deb is 175, key is test_row_0/A:col10/1732372013181/Put/seqid=0 2024-11-23T14:26:53,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742255_1431 (size=31105) 2024-11-23T14:26:53,780 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=248, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/A/a39f40eee3d449d9983773eb0ac23deb 2024-11-23T14:26:53,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/B/d48fc12ff131488fa5d791a6d18bf905 is 50, key is test_row_0/B:col10/1732372013181/Put/seqid=0 2024-11-23T14:26:53,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742256_1432 (size=12151) 2024-11-23T14:26:53,828 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/B/d48fc12ff131488fa5d791a6d18bf905 2024-11-23T14:26:53,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on fa04d9e7f6ce6ae3964ded9c7627848b 2024-11-23T14:26:53,844 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. as already flushing 2024-11-23T14:26:53,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/C/2755ecb8e6f1495e9383084a9665e257 is 50, key is test_row_0/C:col10/1732372013181/Put/seqid=0 2024-11-23T14:26:53,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742257_1433 (size=12151) 2024-11-23T14:26:53,868 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/C/2755ecb8e6f1495e9383084a9665e257 2024-11-23T14:26:53,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/A/a39f40eee3d449d9983773eb0ac23deb as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/a39f40eee3d449d9983773eb0ac23deb 2024-11-23T14:26:53,877 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/a39f40eee3d449d9983773eb0ac23deb, entries=150, sequenceid=248, filesize=30.4 K 2024-11-23T14:26:53,877 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:53,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:53,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32856 deadline: 1732372073863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:53,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32896 deadline: 1732372073864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:53,878 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:53,878 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:53,878 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:53,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:53,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32848 deadline: 1732372073864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:53,879 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:53,879 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:53,879 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:53,880 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:53,880 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:53,880 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:53,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-23T14:26:53,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/B/d48fc12ff131488fa5d791a6d18bf905 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/d48fc12ff131488fa5d791a6d18bf905 2024-11-23T14:26:53,889 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/d48fc12ff131488fa5d791a6d18bf905, entries=150, sequenceid=248, filesize=11.9 K 2024-11-23T14:26:53,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/C/2755ecb8e6f1495e9383084a9665e257 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/2755ecb8e6f1495e9383084a9665e257 2024-11-23T14:26:53,893 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:53,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32838 deadline: 1732372073870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:53,893 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:53,893 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:53,893 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:53,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:53,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32882 deadline: 1732372073871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:53,894 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/2755ecb8e6f1495e9383084a9665e257, entries=150, sequenceid=248, filesize=11.9 K 2024-11-23T14:26:53,895 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:53,895 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:53,895 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:53,896 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for fa04d9e7f6ce6ae3964ded9c7627848b in 179ms, sequenceid=248, compaction requested=true 2024-11-23T14:26:53,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for fa04d9e7f6ce6ae3964ded9c7627848b: 2024-11-23T14:26:53,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:53,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-23T14:26:53,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-23T14:26:53,900 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-23T14:26:53,900 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 334 msec 2024-11-23T14:26:53,902 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 340 msec 2024-11-23T14:26:53,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on fa04d9e7f6ce6ae3964ded9c7627848b 2024-11-23T14:26:53,983 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T14:26:53,983 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fa04d9e7f6ce6ae3964ded9c7627848b 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-23T14:26:53,998 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123ee89fb4721e14893acb020bd8709195f_fa04d9e7f6ce6ae3964ded9c7627848b is 50, key is test_row_0/A:col10/1732372013859/Put/seqid=0 2024-11-23T14:26:54,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742258_1434 (size=14994) 2024-11-23T14:26:54,031 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:26:54,035 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123ee89fb4721e14893acb020bd8709195f_fa04d9e7f6ce6ae3964ded9c7627848b to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123ee89fb4721e14893acb020bd8709195f_fa04d9e7f6ce6ae3964ded9c7627848b 2024-11-23T14:26:54,036 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/A/a8b8b4248f70468c97b69bab16a8fe84, store: [table=TestAcidGuarantees family=A region=fa04d9e7f6ce6ae3964ded9c7627848b] 2024-11-23T14:26:54,036 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/A/a8b8b4248f70468c97b69bab16a8fe84 is 175, key is test_row_0/A:col10/1732372013859/Put/seqid=0 2024-11-23T14:26:54,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742259_1435 (size=39949) 2024-11-23T14:26:54,058 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=264, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/A/a8b8b4248f70468c97b69bab16a8fe84 2024-11-23T14:26:54,061 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:54,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32882 deadline: 1732372074044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:54,061 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:54,061 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:54,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32896 deadline: 1732372074045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:54,061 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:54,061 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:54,062 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:54,062 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:54,062 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:54,062 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:54,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32856 deadline: 1732372074047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:54,063 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:54,063 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:54,063 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:54,069 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:54,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32838 deadline: 1732372074052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:54,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:54,070 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:54,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32848 deadline: 1732372074053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:54,070 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:54,070 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:54,070 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:54,070 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:54,070 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:54,076 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/B/3c33325793c643e1b24eba94ac33aec5 is 50, key is test_row_0/B:col10/1732372013859/Put/seqid=0 2024-11-23T14:26:54,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742260_1436 (size=12301) 2024-11-23T14:26:54,169 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/B/7337d9279f954897bc434aed573f8aec as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/7337d9279f954897bc434aed573f8aec 2024-11-23T14:26:54,180 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fa04d9e7f6ce6ae3964ded9c7627848b/B of fa04d9e7f6ce6ae3964ded9c7627848b into 7337d9279f954897bc434aed573f8aec(size=12.4 K), total size for store is 24.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:26:54,180 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fa04d9e7f6ce6ae3964ded9c7627848b: 2024-11-23T14:26:54,180 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., storeName=fa04d9e7f6ce6ae3964ded9c7627848b/B, priority=13, startTime=1732372013698; duration=0sec 2024-11-23T14:26:54,181 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/A/936fc6eca3424aaeb9dc7d75ec6f0400 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/936fc6eca3424aaeb9dc7d75ec6f0400 2024-11-23T14:26:54,185 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:26:54,185 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fa04d9e7f6ce6ae3964ded9c7627848b:B 2024-11-23T14:26:54,186 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:54,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32896 deadline: 1732372074170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:54,186 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:54,186 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:54,186 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:54,186 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:54,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32882 deadline: 1732372074171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:54,187 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:54,187 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:54,187 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:54,187 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:54,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32856 deadline: 1732372074171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:54,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:54,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:54,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:54,187 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:54,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32838 deadline: 1732372074180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:54,188 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T14:26:54,188 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:54,188 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:54,188 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:54,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:54,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32848 deadline: 1732372074181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:54,189 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:54,189 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:54,189 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:54,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-23T14:26:54,190 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T14:26:54,191 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-23T14:26:54,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-11-23T14:26:54,192 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T14:26:54,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-23T14:26:54,193 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T14:26:54,193 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T14:26:54,197 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fa04d9e7f6ce6ae3964ded9c7627848b/A of fa04d9e7f6ce6ae3964ded9c7627848b into 936fc6eca3424aaeb9dc7d75ec6f0400(size=30.9 K), total size for store is 61.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:26:54,197 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fa04d9e7f6ce6ae3964ded9c7627848b: 2024-11-23T14:26:54,197 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., storeName=fa04d9e7f6ce6ae3964ded9c7627848b/A, priority=13, startTime=1732372013698; duration=0sec 2024-11-23T14:26:54,197 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:26:54,197 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fa04d9e7f6ce6ae3964ded9c7627848b:A 2024-11-23T14:26:54,198 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T14:26:54,198 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1541): fa04d9e7f6ce6ae3964ded9c7627848b/C is initiating minor compaction (all files) 2024-11-23T14:26:54,198 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fa04d9e7f6ce6ae3964ded9c7627848b/C in TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:54,198 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/44cd185037fa4277a4a53d56e1672f3d, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/d4e974bb019e4ef8a87d3b758859d8a9, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/65bb71c263964fdeb0fa0cccdf8b936c, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/2755ecb8e6f1495e9383084a9665e257] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp, totalSize=47.9 K 2024-11-23T14:26:54,198 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 44cd185037fa4277a4a53d56e1672f3d, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732372010748 2024-11-23T14:26:54,199 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting d4e974bb019e4ef8a87d3b758859d8a9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1732372011931 2024-11-23T14:26:54,199 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 65bb71c263964fdeb0fa0cccdf8b936c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1732372013105 2024-11-23T14:26:54,199 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 2755ecb8e6f1495e9383084a9665e257, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1732372013160 2024-11-23T14:26:54,213 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fa04d9e7f6ce6ae3964ded9c7627848b#C#compaction#370 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:26:54,214 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/C/c74d35807c714464947596ce87d0824a is 50, key is test_row_0/C:col10/1732372013181/Put/seqid=0 2024-11-23T14:26:54,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742261_1437 (size=12731) 2024-11-23T14:26:54,253 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/C/c74d35807c714464947596ce87d0824a as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/c74d35807c714464947596ce87d0824a 2024-11-23T14:26:54,260 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fa04d9e7f6ce6ae3964ded9c7627848b/C of fa04d9e7f6ce6ae3964ded9c7627848b into c74d35807c714464947596ce87d0824a(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:26:54,260 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fa04d9e7f6ce6ae3964ded9c7627848b: 2024-11-23T14:26:54,260 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., storeName=fa04d9e7f6ce6ae3964ded9c7627848b/C, priority=12, startTime=1732372013698; duration=0sec 2024-11-23T14:26:54,260 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:26:54,260 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fa04d9e7f6ce6ae3964ded9c7627848b:C 2024-11-23T14:26:54,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-23T14:26:54,346 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-23T14:26:54,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:54,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. as already flushing 2024-11-23T14:26:54,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:54,347 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:54,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:54,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:54,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:54,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32856 deadline: 1732372074391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:54,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:54,397 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:54,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:54,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:54,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32896 deadline: 1732372074391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:54,397 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:54,397 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:54,397 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:54,397 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:54,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32882 deadline: 1732372074391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:54,397 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:54,397 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:54,397 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:54,409 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:54,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32848 deadline: 1732372074400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:54,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:54,410 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:54,410 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:54,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32838 deadline: 1732372074400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:54,410 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:54,410 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:54,410 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:54,410 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:54,504 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-23T14:26:54,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:54,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. as already flushing 2024-11-23T14:26:54,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:54,505 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:54,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:54,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:54,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-23T14:26:54,514 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/B/3c33325793c643e1b24eba94ac33aec5 2024-11-23T14:26:54,525 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/C/174b6d59aec34f549fae7965ad77926e is 50, key is test_row_0/C:col10/1732372013859/Put/seqid=0 2024-11-23T14:26:54,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742262_1438 (size=12301) 2024-11-23T14:26:54,564 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/C/174b6d59aec34f549fae7965ad77926e 2024-11-23T14:26:54,571 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/A/a8b8b4248f70468c97b69bab16a8fe84 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/a8b8b4248f70468c97b69bab16a8fe84 2024-11-23T14:26:54,575 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/a8b8b4248f70468c97b69bab16a8fe84, entries=200, sequenceid=264, filesize=39.0 K 2024-11-23T14:26:54,580 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/B/3c33325793c643e1b24eba94ac33aec5 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/3c33325793c643e1b24eba94ac33aec5 2024-11-23T14:26:54,590 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/3c33325793c643e1b24eba94ac33aec5, entries=150, sequenceid=264, filesize=12.0 K 2024-11-23T14:26:54,591 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/C/174b6d59aec34f549fae7965ad77926e as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/174b6d59aec34f549fae7965ad77926e 2024-11-23T14:26:54,598 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/174b6d59aec34f549fae7965ad77926e, entries=150, sequenceid=264, filesize=12.0 K 2024-11-23T14:26:54,598 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for fa04d9e7f6ce6ae3964ded9c7627848b in 615ms, sequenceid=264, compaction requested=true 2024-11-23T14:26:54,599 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fa04d9e7f6ce6ae3964ded9c7627848b: 2024-11-23T14:26:54,599 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:26:54,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fa04d9e7f6ce6ae3964ded9c7627848b:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T14:26:54,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:26:54,599 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:26:54,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fa04d9e7f6ce6ae3964ded9c7627848b:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T14:26:54,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:26:54,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fa04d9e7f6ce6ae3964ded9c7627848b:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T14:26:54,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:26:54,600 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:26:54,601 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1541): fa04d9e7f6ce6ae3964ded9c7627848b/B is initiating minor compaction (all files) 2024-11-23T14:26:54,601 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fa04d9e7f6ce6ae3964ded9c7627848b/B in TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:54,601 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/7337d9279f954897bc434aed573f8aec, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/d48fc12ff131488fa5d791a6d18bf905, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/3c33325793c643e1b24eba94ac33aec5] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp, totalSize=36.3 K 2024-11-23T14:26:54,601 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102705 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:26:54,601 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1541): fa04d9e7f6ce6ae3964ded9c7627848b/A is initiating minor compaction (all files) 2024-11-23T14:26:54,601 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fa04d9e7f6ce6ae3964ded9c7627848b/A in TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:54,601 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/936fc6eca3424aaeb9dc7d75ec6f0400, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/a39f40eee3d449d9983773eb0ac23deb, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/a8b8b4248f70468c97b69bab16a8fe84] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp, totalSize=100.3 K 2024-11-23T14:26:54,601 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:54,601 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. files: [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/936fc6eca3424aaeb9dc7d75ec6f0400, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/a39f40eee3d449d9983773eb0ac23deb, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/a8b8b4248f70468c97b69bab16a8fe84] 2024-11-23T14:26:54,602 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 7337d9279f954897bc434aed573f8aec, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1732372013105 2024-11-23T14:26:54,602 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 936fc6eca3424aaeb9dc7d75ec6f0400, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1732372013105 2024-11-23T14:26:54,602 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting d48fc12ff131488fa5d791a6d18bf905, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1732372013160 2024-11-23T14:26:54,604 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting a39f40eee3d449d9983773eb0ac23deb, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1732372013160 2024-11-23T14:26:54,605 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c33325793c643e1b24eba94ac33aec5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1732372013859 2024-11-23T14:26:54,605 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting a8b8b4248f70468c97b69bab16a8fe84, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1732372013859 2024-11-23T14:26:54,619 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fa04d9e7f6ce6ae3964ded9c7627848b] 2024-11-23T14:26:54,620 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fa04d9e7f6ce6ae3964ded9c7627848b#B#compaction#372 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:26:54,620 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/B/ee1c54e248dd409b823f78ab872a0056 is 50, key is test_row_0/B:col10/1732372013859/Put/seqid=0 2024-11-23T14:26:54,623 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411230e84b869c91446f598f9b365e05d210e_fa04d9e7f6ce6ae3964ded9c7627848b store=[table=TestAcidGuarantees family=A region=fa04d9e7f6ce6ae3964ded9c7627848b] 2024-11-23T14:26:54,625 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411230e84b869c91446f598f9b365e05d210e_fa04d9e7f6ce6ae3964ded9c7627848b, store=[table=TestAcidGuarantees family=A region=fa04d9e7f6ce6ae3964ded9c7627848b] 2024-11-23T14:26:54,625 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411230e84b869c91446f598f9b365e05d210e_fa04d9e7f6ce6ae3964ded9c7627848b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fa04d9e7f6ce6ae3964ded9c7627848b] 2024-11-23T14:26:54,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742263_1439 (size=12949) 2024-11-23T14:26:54,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742264_1440 (size=4469) 2024-11-23T14:26:54,647 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fa04d9e7f6ce6ae3964ded9c7627848b#A#compaction#373 average throughput is 0.87 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:26:54,647 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/A/0a1bd3224fbd4c4d83d15d5d071cd781 is 175, key is test_row_0/A:col10/1732372013859/Put/seqid=0 2024-11-23T14:26:54,652 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/B/ee1c54e248dd409b823f78ab872a0056 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/ee1c54e248dd409b823f78ab872a0056 2024-11-23T14:26:54,658 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fa04d9e7f6ce6ae3964ded9c7627848b/B of fa04d9e7f6ce6ae3964ded9c7627848b into ee1c54e248dd409b823f78ab872a0056(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:26:54,658 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fa04d9e7f6ce6ae3964ded9c7627848b: 2024-11-23T14:26:54,658 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., storeName=fa04d9e7f6ce6ae3964ded9c7627848b/B, priority=13, startTime=1732372014599; duration=0sec 2024-11-23T14:26:54,658 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:26:54,658 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fa04d9e7f6ce6ae3964ded9c7627848b:B 2024-11-23T14:26:54,658 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-23T14:26:54,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742265_1441 (size=31903) 2024-11-23T14:26:54,659 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-23T14:26:54,659 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-23T14:26:54,659 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. because compaction request was cancelled 2024-11-23T14:26:54,659 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fa04d9e7f6ce6ae3964ded9c7627848b:C 2024-11-23T14:26:54,661 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-23T14:26:54,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:54,661 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing fa04d9e7f6ce6ae3964ded9c7627848b 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-23T14:26:54,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411232cc522e57250483f87e01c1e6a6c5e5a_fa04d9e7f6ce6ae3964ded9c7627848b is 50, key is test_row_0/A:col10/1732372014045/Put/seqid=0 2024-11-23T14:26:54,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742266_1442 (size=12454) 2024-11-23T14:26:54,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on fa04d9e7f6ce6ae3964ded9c7627848b 2024-11-23T14:26:54,707 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. as already flushing 2024-11-23T14:26:54,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:54,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32896 deadline: 1732372074731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:54,738 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:54,738 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:54,738 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:54,745 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:54,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32838 deadline: 1732372074736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:54,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:54,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32882 deadline: 1732372074736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:54,746 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:54,746 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:54,746 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:54,746 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:54,746 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:54,746 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:54,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:54,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32856 deadline: 1732372074736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:54,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:54,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:54,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:54,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:54,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32848 deadline: 1732372074737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:54,748 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:54,748 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:54,748 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:54,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-23T14:26:54,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:54,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32896 deadline: 1732372074840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:54,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:54,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:54,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:54,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:54,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32838 deadline: 1732372074850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:54,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:54,853 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:54,853 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:54,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32856 deadline: 1732372074850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:54,853 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:54,854 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:54,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32882 deadline: 1732372074851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:54,854 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:54,854 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:54,854 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:54,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32848 deadline: 1732372074851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:54,854 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:54,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:54,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:54,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:54,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:54,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:54,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:55,058 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:55,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32896 deadline: 1732372075050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:55,059 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:55,059 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:55,059 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:55,064 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/A/0a1bd3224fbd4c4d83d15d5d071cd781 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/0a1bd3224fbd4c4d83d15d5d071cd781 2024-11-23T14:26:55,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:55,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32848 deadline: 1732372075060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:55,065 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:55,065 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:55,065 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:55,072 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:55,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32838 deadline: 1732372075061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:55,072 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:55,072 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:55,072 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:55,072 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:55,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32856 deadline: 1732372075061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:55,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:55,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:55,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:55,073 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:55,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32882 deadline: 1732372075061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:55,073 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fa04d9e7f6ce6ae3964ded9c7627848b/A of fa04d9e7f6ce6ae3964ded9c7627848b into 0a1bd3224fbd4c4d83d15d5d071cd781(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:26:55,073 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fa04d9e7f6ce6ae3964ded9c7627848b: 2024-11-23T14:26:55,073 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:55,073 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:55,073 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:55,073 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., storeName=fa04d9e7f6ce6ae3964ded9c7627848b/A, priority=13, startTime=1732372014599; duration=0sec 2024-11-23T14:26:55,073 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:26:55,073 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fa04d9e7f6ce6ae3964ded9c7627848b:A 2024-11-23T14:26:55,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:26:55,083 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411232cc522e57250483f87e01c1e6a6c5e5a_fa04d9e7f6ce6ae3964ded9c7627848b to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411232cc522e57250483f87e01c1e6a6c5e5a_fa04d9e7f6ce6ae3964ded9c7627848b 2024-11-23T14:26:55,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/A/cd45d1ce987743d380ac514180bcac0a, store: [table=TestAcidGuarantees family=A region=fa04d9e7f6ce6ae3964ded9c7627848b] 2024-11-23T14:26:55,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/A/cd45d1ce987743d380ac514180bcac0a is 175, key is test_row_0/A:col10/1732372014045/Put/seqid=0 2024-11-23T14:26:55,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742267_1443 (size=31255) 2024-11-23T14:26:55,099 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=288, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/A/cd45d1ce987743d380ac514180bcac0a 2024-11-23T14:26:55,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/B/3a388032a98f4be587e08cba96858dc4 is 50, key is test_row_0/B:col10/1732372014045/Put/seqid=0 2024-11-23T14:26:55,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742268_1444 (size=12301) 2024-11-23T14:26:55,128 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/B/3a388032a98f4be587e08cba96858dc4 2024-11-23T14:26:55,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/C/859aa3cc415444638298019da215234a is 50, key is test_row_0/C:col10/1732372014045/Put/seqid=0 2024-11-23T14:26:55,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742269_1445 (size=12301) 2024-11-23T14:26:55,170 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/C/859aa3cc415444638298019da215234a 2024-11-23T14:26:55,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/A/cd45d1ce987743d380ac514180bcac0a as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/cd45d1ce987743d380ac514180bcac0a 2024-11-23T14:26:55,178 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/cd45d1ce987743d380ac514180bcac0a, entries=150, sequenceid=288, filesize=30.5 K 2024-11-23T14:26:55,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/B/3a388032a98f4be587e08cba96858dc4 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/3a388032a98f4be587e08cba96858dc4 2024-11-23T14:26:55,184 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/3a388032a98f4be587e08cba96858dc4, entries=150, sequenceid=288, filesize=12.0 K 2024-11-23T14:26:55,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/C/859aa3cc415444638298019da215234a as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/859aa3cc415444638298019da215234a 2024-11-23T14:26:55,188 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/859aa3cc415444638298019da215234a, entries=150, sequenceid=288, filesize=12.0 K 2024-11-23T14:26:55,189 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for fa04d9e7f6ce6ae3964ded9c7627848b in 528ms, sequenceid=288, compaction requested=true 2024-11-23T14:26:55,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for fa04d9e7f6ce6ae3964ded9c7627848b: 2024-11-23T14:26:55,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:55,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-11-23T14:26:55,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-11-23T14:26:55,192 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-23T14:26:55,192 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 998 msec 2024-11-23T14:26:55,193 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 1.0010 sec 2024-11-23T14:26:55,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-23T14:26:55,331 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T14:26:55,332 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-23T14:26:55,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-11-23T14:26:55,334 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T14:26:55,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-23T14:26:55,335 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T14:26:55,335 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T14:26:55,375 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T14:26:55,375 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fa04d9e7f6ce6ae3964ded9c7627848b 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-23T14:26:55,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on fa04d9e7f6ce6ae3964ded9c7627848b 2024-11-23T14:26:55,405 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123cf36417990e54ab5a7da49c1002c7511_fa04d9e7f6ce6ae3964ded9c7627848b is 50, key is test_row_0/A:col10/1732372015372/Put/seqid=0 2024-11-23T14:26:55,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742270_1446 (size=14994) 2024-11-23T14:26:55,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-23T14:26:55,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:55,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32856 deadline: 1732372075454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:55,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:55,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32896 deadline: 1732372075454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:55,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:55,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:55,469 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:55,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:55,469 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:55,469 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:55,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:55,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32848 deadline: 1732372075457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:55,472 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:55,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32838 deadline: 1732372075468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:55,473 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:55,473 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:55,473 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:55,473 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:55,473 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:55,473 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:55,477 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:55,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32882 deadline: 1732372075469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:55,477 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:55,478 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:55,478 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:55,488 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-23T14:26:55,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:55,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. as already flushing 2024-11-23T14:26:55,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:55,488 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:55,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:55,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:55,590 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:55,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32896 deadline: 1732372075581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:55,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:55,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:55,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:55,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:55,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32856 deadline: 1732372075581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:55,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:55,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:55,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:55,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:55,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32882 deadline: 1732372075581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:55,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:55,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:55,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:55,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:55,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32838 deadline: 1732372075581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:55,592 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:55,592 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:55,592 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:55,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:55,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32848 deadline: 1732372075581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:55,593 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:55,593 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:55,593 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:55,641 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-23T14:26:55,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:55,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. as already flushing 2024-11-23T14:26:55,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:55,641 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:55,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:55,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:55,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-23T14:26:55,799 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-23T14:26:55,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:55,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. as already flushing 2024-11-23T14:26:55,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:55,800 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:55,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:55,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:55,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:55,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32896 deadline: 1732372075801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:55,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:55,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32848 deadline: 1732372075801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:55,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:55,808 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:55,808 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:55,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32838 deadline: 1732372075801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:55,808 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:55,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:55,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32856 deadline: 1732372075801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:55,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:55,809 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:55,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:55,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:55,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32882 deadline: 1732372075801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:55,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:55,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:55,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:55,809 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:55,809 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:55,809 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:55,809 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:55,809 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:55,809 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:55,816 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:26:55,819 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123cf36417990e54ab5a7da49c1002c7511_fa04d9e7f6ce6ae3964ded9c7627848b to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123cf36417990e54ab5a7da49c1002c7511_fa04d9e7f6ce6ae3964ded9c7627848b 2024-11-23T14:26:55,820 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/A/f0fa6dc573ef455cb66aaea113359951, store: [table=TestAcidGuarantees family=A region=fa04d9e7f6ce6ae3964ded9c7627848b] 2024-11-23T14:26:55,821 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/A/f0fa6dc573ef455cb66aaea113359951 is 175, key is test_row_0/A:col10/1732372015372/Put/seqid=0 2024-11-23T14:26:55,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742271_1447 (size=39949) 2024-11-23T14:26:55,836 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=306, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/A/f0fa6dc573ef455cb66aaea113359951 2024-11-23T14:26:55,852 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/B/575a30ad120e4a5c96a4631bc6c69330 is 50, key is test_row_0/B:col10/1732372015372/Put/seqid=0 2024-11-23T14:26:55,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742272_1448 (size=12301) 2024-11-23T14:26:55,878 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/B/575a30ad120e4a5c96a4631bc6c69330 2024-11-23T14:26:55,890 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/C/ebbfb4822a634c2f866340149d489832 is 50, key is test_row_0/C:col10/1732372015372/Put/seqid=0 2024-11-23T14:26:55,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742273_1449 (size=12301) 2024-11-23T14:26:55,913 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/C/ebbfb4822a634c2f866340149d489832 2024-11-23T14:26:55,919 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/A/f0fa6dc573ef455cb66aaea113359951 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/f0fa6dc573ef455cb66aaea113359951 2024-11-23T14:26:55,925 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/f0fa6dc573ef455cb66aaea113359951, entries=200, sequenceid=306, filesize=39.0 K 2024-11-23T14:26:55,925 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/B/575a30ad120e4a5c96a4631bc6c69330 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/575a30ad120e4a5c96a4631bc6c69330 2024-11-23T14:26:55,929 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/575a30ad120e4a5c96a4631bc6c69330, entries=150, sequenceid=306, filesize=12.0 K 2024-11-23T14:26:55,930 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/C/ebbfb4822a634c2f866340149d489832 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/ebbfb4822a634c2f866340149d489832 2024-11-23T14:26:55,933 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/ebbfb4822a634c2f866340149d489832, entries=150, sequenceid=306, filesize=12.0 K 2024-11-23T14:26:55,934 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for fa04d9e7f6ce6ae3964ded9c7627848b in 559ms, sequenceid=306, compaction requested=true 2024-11-23T14:26:55,934 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fa04d9e7f6ce6ae3964ded9c7627848b: 2024-11-23T14:26:55,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fa04d9e7f6ce6ae3964ded9c7627848b:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T14:26:55,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:26:55,934 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:26:55,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fa04d9e7f6ce6ae3964ded9c7627848b:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T14:26:55,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:26:55,934 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:26:55,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fa04d9e7f6ce6ae3964ded9c7627848b:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T14:26:55,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:26:55,939 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:26:55,940 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1541): fa04d9e7f6ce6ae3964ded9c7627848b/B is initiating minor compaction (all files) 2024-11-23T14:26:55,940 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fa04d9e7f6ce6ae3964ded9c7627848b/B in TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:55,940 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/ee1c54e248dd409b823f78ab872a0056, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/3a388032a98f4be587e08cba96858dc4, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/575a30ad120e4a5c96a4631bc6c69330] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp, totalSize=36.7 K 2024-11-23T14:26:55,940 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103107 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:26:55,940 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1541): fa04d9e7f6ce6ae3964ded9c7627848b/A is initiating minor compaction (all files) 2024-11-23T14:26:55,940 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fa04d9e7f6ce6ae3964ded9c7627848b/A in TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:55,940 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/0a1bd3224fbd4c4d83d15d5d071cd781, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/cd45d1ce987743d380ac514180bcac0a, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/f0fa6dc573ef455cb66aaea113359951] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp, totalSize=100.7 K 2024-11-23T14:26:55,940 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:55,940 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. files: [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/0a1bd3224fbd4c4d83d15d5d071cd781, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/cd45d1ce987743d380ac514180bcac0a, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/f0fa6dc573ef455cb66aaea113359951] 2024-11-23T14:26:55,941 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting ee1c54e248dd409b823f78ab872a0056, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1732372013859 2024-11-23T14:26:55,943 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a1bd3224fbd4c4d83d15d5d071cd781, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1732372013859 2024-11-23T14:26:55,943 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a388032a98f4be587e08cba96858dc4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1732372014045 2024-11-23T14:26:55,943 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd45d1ce987743d380ac514180bcac0a, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1732372014045 2024-11-23T14:26:55,943 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 575a30ad120e4a5c96a4631bc6c69330, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1732372014729 2024-11-23T14:26:55,943 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting f0fa6dc573ef455cb66aaea113359951, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1732372014729 2024-11-23T14:26:55,954 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fa04d9e7f6ce6ae3964ded9c7627848b#B#compaction#380 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:26:55,954 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-23T14:26:55,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:55,954 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2902): Flushing fa04d9e7f6ce6ae3964ded9c7627848b 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-23T14:26:55,954 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/B/394c8fcd86f546fa9c6ee1440f209d0e is 50, key is test_row_0/B:col10/1732372015372/Put/seqid=0 2024-11-23T14:26:55,955 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fa04d9e7f6ce6ae3964ded9c7627848b] 2024-11-23T14:26:55,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-23T14:26:55,969 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411233e0b06a36fdf4d16a3b650844b3e6c57_fa04d9e7f6ce6ae3964ded9c7627848b store=[table=TestAcidGuarantees family=A region=fa04d9e7f6ce6ae3964ded9c7627848b] 2024-11-23T14:26:55,970 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411233e0b06a36fdf4d16a3b650844b3e6c57_fa04d9e7f6ce6ae3964ded9c7627848b, store=[table=TestAcidGuarantees family=A region=fa04d9e7f6ce6ae3964ded9c7627848b] 2024-11-23T14:26:55,971 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411233e0b06a36fdf4d16a3b650844b3e6c57_fa04d9e7f6ce6ae3964ded9c7627848b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fa04d9e7f6ce6ae3964ded9c7627848b] 2024-11-23T14:26:55,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742274_1450 (size=13051) 2024-11-23T14:26:55,981 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/B/394c8fcd86f546fa9c6ee1440f209d0e as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/394c8fcd86f546fa9c6ee1440f209d0e 2024-11-23T14:26:55,984 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fa04d9e7f6ce6ae3964ded9c7627848b/B of fa04d9e7f6ce6ae3964ded9c7627848b into 394c8fcd86f546fa9c6ee1440f209d0e(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:26:55,984 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fa04d9e7f6ce6ae3964ded9c7627848b: 2024-11-23T14:26:55,984 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., storeName=fa04d9e7f6ce6ae3964ded9c7627848b/B, priority=13, startTime=1732372015934; duration=0sec 2024-11-23T14:26:55,984 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:26:55,984 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fa04d9e7f6ce6ae3964ded9c7627848b:B 2024-11-23T14:26:55,984 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T14:26:55,985 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49634 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T14:26:55,985 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1541): fa04d9e7f6ce6ae3964ded9c7627848b/C is initiating minor compaction (all files) 2024-11-23T14:26:55,986 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fa04d9e7f6ce6ae3964ded9c7627848b/C in TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:55,986 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/c74d35807c714464947596ce87d0824a, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/174b6d59aec34f549fae7965ad77926e, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/859aa3cc415444638298019da215234a, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/ebbfb4822a634c2f866340149d489832] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp, totalSize=48.5 K 2024-11-23T14:26:55,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123787b54a1f8ee4a04a73f058d85bb9f73_fa04d9e7f6ce6ae3964ded9c7627848b is 50, key is test_row_0/A:col10/1732372015466/Put/seqid=0 2024-11-23T14:26:55,986 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting c74d35807c714464947596ce87d0824a, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1732372013160 2024-11-23T14:26:55,986 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 174b6d59aec34f549fae7965ad77926e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1732372013859 2024-11-23T14:26:55,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742275_1451 (size=4469) 2024-11-23T14:26:55,987 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 859aa3cc415444638298019da215234a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1732372014045 2024-11-23T14:26:55,987 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting ebbfb4822a634c2f866340149d489832, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1732372014729 2024-11-23T14:26:55,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742276_1452 (size=12454) 2024-11-23T14:26:56,002 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fa04d9e7f6ce6ae3964ded9c7627848b#C#compaction#383 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:26:56,002 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/C/1be829ff64c74c919b1781ea5e873d07 is 50, key is test_row_0/C:col10/1732372015372/Put/seqid=0 2024-11-23T14:26:56,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742277_1453 (size=13017) 2024-11-23T14:26:56,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on fa04d9e7f6ce6ae3964ded9c7627848b 2024-11-23T14:26:56,126 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. as already flushing 2024-11-23T14:26:56,158 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:56,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32838 deadline: 1732372076144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:56,158 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:56,158 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:56,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32848 deadline: 1732372076144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:56,158 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:56,158 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:56,158 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:56,158 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:56,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32856 deadline: 1732372076155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:56,158 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:56,158 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:56,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:56,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:56,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:56,168 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:56,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32896 deadline: 1732372076158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:56,168 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:56,168 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:56,168 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:56,168 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:56,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32882 deadline: 1732372076160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:56,168 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:56,168 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:56,168 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:56,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:56,269 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:56,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32838 deadline: 1732372076260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:56,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32856 deadline: 1732372076260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:56,270 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:56,270 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:56,270 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:56,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:56,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:56,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:56,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:56,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32848 deadline: 1732372076270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:56,281 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:56,281 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:56,281 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:56,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:56,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32896 deadline: 1732372076270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:56,281 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:56,281 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:56,281 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:56,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:56,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32882 deadline: 1732372076272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:56,281 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:56,281 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:56,282 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:56,388 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fa04d9e7f6ce6ae3964ded9c7627848b#A#compaction#381 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:26:56,389 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/A/6ed354cb0df5484391030953e63c5e13 is 175, key is test_row_0/A:col10/1732372015372/Put/seqid=0 2024-11-23T14:26:56,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:26:56,399 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123787b54a1f8ee4a04a73f058d85bb9f73_fa04d9e7f6ce6ae3964ded9c7627848b to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123787b54a1f8ee4a04a73f058d85bb9f73_fa04d9e7f6ce6ae3964ded9c7627848b 2024-11-23T14:26:56,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/A/647ecb3218404e4b846a6c6b1d6231e7, store: [table=TestAcidGuarantees family=A region=fa04d9e7f6ce6ae3964ded9c7627848b] 2024-11-23T14:26:56,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/A/647ecb3218404e4b846a6c6b1d6231e7 is 175, key is test_row_0/A:col10/1732372015466/Put/seqid=0 2024-11-23T14:26:56,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742278_1454 (size=32005) 2024-11-23T14:26:56,429 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/C/1be829ff64c74c919b1781ea5e873d07 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/1be829ff64c74c919b1781ea5e873d07 2024-11-23T14:26:56,435 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fa04d9e7f6ce6ae3964ded9c7627848b/C of fa04d9e7f6ce6ae3964ded9c7627848b into 1be829ff64c74c919b1781ea5e873d07(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:26:56,435 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fa04d9e7f6ce6ae3964ded9c7627848b: 2024-11-23T14:26:56,435 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., storeName=fa04d9e7f6ce6ae3964ded9c7627848b/C, priority=12, startTime=1732372015934; duration=0sec 2024-11-23T14:26:56,435 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:26:56,435 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fa04d9e7f6ce6ae3964ded9c7627848b:C 2024-11-23T14:26:56,435 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/A/6ed354cb0df5484391030953e63c5e13 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/6ed354cb0df5484391030953e63c5e13 2024-11-23T14:26:56,448 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fa04d9e7f6ce6ae3964ded9c7627848b/A of fa04d9e7f6ce6ae3964ded9c7627848b into 6ed354cb0df5484391030953e63c5e13(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:26:56,448 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fa04d9e7f6ce6ae3964ded9c7627848b: 2024-11-23T14:26:56,448 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., storeName=fa04d9e7f6ce6ae3964ded9c7627848b/A, priority=13, startTime=1732372015934; duration=0sec 2024-11-23T14:26:56,448 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:26:56,449 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fa04d9e7f6ce6ae3964ded9c7627848b:A 2024-11-23T14:26:56,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-23T14:26:56,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742279_1455 (size=31255) 2024-11-23T14:26:56,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:56,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32856 deadline: 1732372076482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:56,495 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:56,495 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:56,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:56,495 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:56,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32838 deadline: 1732372076483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:56,496 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:56,496 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:56,496 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:56,500 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:56,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32848 deadline: 1732372076490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:56,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:56,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32896 deadline: 1732372076490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:56,501 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:56,501 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:56,501 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:56,501 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:56,501 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:56,501 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:56,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:56,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32882 deadline: 1732372076491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:56,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:56,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:56,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:56,804 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:56,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32856 deadline: 1732372076800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:56,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:56,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:56,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:56,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:56,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32838 deadline: 1732372076801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:56,805 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:56,805 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:56,805 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:56,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:56,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32848 deadline: 1732372076810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:56,820 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:56,820 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:56,820 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:56,820 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:56,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32896 deadline: 1732372076810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:56,820 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:56,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:56,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:56,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:56,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32882 deadline: 1732372076810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:56,820 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:56,820 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:56,820 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:56,877 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=325, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/A/647ecb3218404e4b846a6c6b1d6231e7 2024-11-23T14:26:56,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/B/3ddbcb1a226d449197f0ae5962faf468 is 50, key is test_row_0/B:col10/1732372015466/Put/seqid=0 2024-11-23T14:26:56,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742280_1456 (size=12301) 2024-11-23T14:26:56,947 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/B/3ddbcb1a226d449197f0ae5962faf468 2024-11-23T14:26:56,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/C/5fec32ffb0ba41ef8b17f29acc847807 is 50, key is test_row_0/C:col10/1732372015466/Put/seqid=0 2024-11-23T14:26:56,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742281_1457 (size=12301) 2024-11-23T14:26:56,959 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/C/5fec32ffb0ba41ef8b17f29acc847807 2024-11-23T14:26:56,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/A/647ecb3218404e4b846a6c6b1d6231e7 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/647ecb3218404e4b846a6c6b1d6231e7 2024-11-23T14:26:56,968 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/647ecb3218404e4b846a6c6b1d6231e7, entries=150, sequenceid=325, filesize=30.5 K 2024-11-23T14:26:56,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/B/3ddbcb1a226d449197f0ae5962faf468 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/3ddbcb1a226d449197f0ae5962faf468 2024-11-23T14:26:56,981 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/3ddbcb1a226d449197f0ae5962faf468, entries=150, sequenceid=325, filesize=12.0 K 2024-11-23T14:26:56,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/C/5fec32ffb0ba41ef8b17f29acc847807 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/5fec32ffb0ba41ef8b17f29acc847807 2024-11-23T14:26:56,987 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/5fec32ffb0ba41ef8b17f29acc847807, entries=150, sequenceid=325, filesize=12.0 K 2024-11-23T14:26:56,988 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for fa04d9e7f6ce6ae3964ded9c7627848b in 1034ms, sequenceid=325, compaction requested=false 2024-11-23T14:26:56,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for fa04d9e7f6ce6ae3964ded9c7627848b: 2024-11-23T14:26:56,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:56,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-11-23T14:26:56,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-11-23T14:26:56,991 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-23T14:26:56,991 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6540 sec 2024-11-23T14:26:56,993 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 1.6600 sec 2024-11-23T14:26:57,317 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T14:26:57,317 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fa04d9e7f6ce6ae3964ded9c7627848b 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-23T14:26:57,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on fa04d9e7f6ce6ae3964ded9c7627848b 2024-11-23T14:26:57,338 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123e1161c19b6cf4d74abb8af1da63f0109_fa04d9e7f6ce6ae3964ded9c7627848b is 50, key is test_row_0/A:col10/1732372017315/Put/seqid=0 2024-11-23T14:26:57,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742282_1458 (size=17534) 2024-11-23T14:26:57,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:57,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32896 deadline: 1732372077359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:57,371 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:57,371 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:57,371 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:57,373 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:57,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32856 deadline: 1732372077361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:57,374 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:57,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:57,374 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:57,374 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:57,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32838 deadline: 1732372077361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:57,374 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:57,374 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:57,374 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:57,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:57,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32848 deadline: 1732372077362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:57,375 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:57,375 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:57,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:57,375 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:57,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32882 deadline: 1732372077370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:57,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:57,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:57,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:57,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-23T14:26:57,480 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T14:26:57,481 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-23T14:26:57,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-11-23T14:26:57,484 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T14:26:57,484 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T14:26:57,484 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T14:26:57,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-23T14:26:57,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:57,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32896 deadline: 1732372077480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:57,489 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:57,489 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:57,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:57,489 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:57,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32856 deadline: 1732372077480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:57,489 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:57,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:57,489 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:57,489 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:57,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32838 deadline: 1732372077481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:57,490 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:57,490 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:57,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:57,490 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:57,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32848 deadline: 1732372077481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:57,490 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:57,490 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:57,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:57,490 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:57,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32882 deadline: 1732372077481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:57,490 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:57,490 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:57,490 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:57,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-23T14:26:57,637 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-23T14:26:57,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:57,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. as already flushing 2024-11-23T14:26:57,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:57,637 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:57,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:57,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:57,708 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:57,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32856 deadline: 1732372077700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:57,708 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:57,708 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:57,709 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:57,709 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:57,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32896 deadline: 1732372077700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:57,709 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:57,709 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:57,709 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:57,709 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:57,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32848 deadline: 1732372077700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:57,709 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:57,709 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:57,709 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:57,709 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:57,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32838 deadline: 1732372077700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:57,710 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:57,710 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:57,710 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:57,710 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:57,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32882 deadline: 1732372077700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:57,710 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:57,710 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:57,710 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:57,770 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:26:57,774 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123e1161c19b6cf4d74abb8af1da63f0109_fa04d9e7f6ce6ae3964ded9c7627848b to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123e1161c19b6cf4d74abb8af1da63f0109_fa04d9e7f6ce6ae3964ded9c7627848b 2024-11-23T14:26:57,775 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/A/50d49f4c642a4f359ef7a5723de02d52, store: [table=TestAcidGuarantees family=A region=fa04d9e7f6ce6ae3964ded9c7627848b] 2024-11-23T14:26:57,776 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/A/50d49f4c642a4f359ef7a5723de02d52 is 175, key is test_row_0/A:col10/1732372017315/Put/seqid=0 2024-11-23T14:26:57,790 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-23T14:26:57,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:57,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. as already flushing 2024-11-23T14:26:57,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:57,791 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:57,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:57,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:57,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-23T14:26:57,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742283_1459 (size=48639) 2024-11-23T14:26:57,817 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=347, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/A/50d49f4c642a4f359ef7a5723de02d52 2024-11-23T14:26:57,835 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/B/3c3a0a9c77344505b66abe8a76b31219 is 50, key is test_row_0/B:col10/1732372017315/Put/seqid=0 2024-11-23T14:26:57,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742284_1460 (size=12301) 2024-11-23T14:26:57,946 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-23T14:26:57,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:57,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. as already flushing 2024-11-23T14:26:57,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:57,946 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:57,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:57,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:58,030 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:58,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32848 deadline: 1732372078021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:58,033 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:58,033 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:58,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:58,033 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:58,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32856 deadline: 1732372078021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:58,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:58,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32896 deadline: 1732372078021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:58,033 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:58,033 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:58,033 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:58,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:58,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32882 deadline: 1732372078021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:58,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:58,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32838 deadline: 1732372078021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:58,034 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:58,035 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:58,035 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:58,037 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:58,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:58,037 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:58,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:58,037 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:58,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:58,099 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-23T14:26:58,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:58,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. as already flushing 2024-11-23T14:26:58,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:58,101 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:58,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:58,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:58,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-23T14:26:58,256 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-23T14:26:58,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:58,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. as already flushing 2024-11-23T14:26:58,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:58,257 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:58,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:58,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:58,278 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/B/3c3a0a9c77344505b66abe8a76b31219 2024-11-23T14:26:58,298 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/C/a7e21b5ba2834ddca414c9b17bf1a105 is 50, key is test_row_0/C:col10/1732372017315/Put/seqid=0 2024-11-23T14:26:58,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742285_1461 (size=12301) 2024-11-23T14:26:58,333 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/C/a7e21b5ba2834ddca414c9b17bf1a105 2024-11-23T14:26:58,339 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/A/50d49f4c642a4f359ef7a5723de02d52 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/50d49f4c642a4f359ef7a5723de02d52 2024-11-23T14:26:58,344 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/50d49f4c642a4f359ef7a5723de02d52, entries=250, sequenceid=347, filesize=47.5 K 2024-11-23T14:26:58,345 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/B/3c3a0a9c77344505b66abe8a76b31219 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/3c3a0a9c77344505b66abe8a76b31219 2024-11-23T14:26:58,349 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/3c3a0a9c77344505b66abe8a76b31219, entries=150, sequenceid=347, filesize=12.0 K 2024-11-23T14:26:58,350 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/C/a7e21b5ba2834ddca414c9b17bf1a105 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/a7e21b5ba2834ddca414c9b17bf1a105 2024-11-23T14:26:58,364 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/a7e21b5ba2834ddca414c9b17bf1a105, entries=150, sequenceid=347, filesize=12.0 K 2024-11-23T14:26:58,366 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for fa04d9e7f6ce6ae3964ded9c7627848b in 1049ms, sequenceid=347, compaction requested=true 2024-11-23T14:26:58,366 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fa04d9e7f6ce6ae3964ded9c7627848b: 2024-11-23T14:26:58,366 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:26:58,367 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111899 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:26:58,367 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1541): fa04d9e7f6ce6ae3964ded9c7627848b/A is initiating minor compaction (all files) 2024-11-23T14:26:58,367 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fa04d9e7f6ce6ae3964ded9c7627848b/A in TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:58,368 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/6ed354cb0df5484391030953e63c5e13, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/647ecb3218404e4b846a6c6b1d6231e7, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/50d49f4c642a4f359ef7a5723de02d52] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp, totalSize=109.3 K 2024-11-23T14:26:58,368 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:58,368 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. files: [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/6ed354cb0df5484391030953e63c5e13, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/647ecb3218404e4b846a6c6b1d6231e7, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/50d49f4c642a4f359ef7a5723de02d52] 2024-11-23T14:26:58,370 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ed354cb0df5484391030953e63c5e13, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1732372014729 2024-11-23T14:26:58,371 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 647ecb3218404e4b846a6c6b1d6231e7, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1732372015450 2024-11-23T14:26:58,372 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 50d49f4c642a4f359ef7a5723de02d52, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1732372016143 2024-11-23T14:26:58,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fa04d9e7f6ce6ae3964ded9c7627848b:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T14:26:58,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:26:58,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fa04d9e7f6ce6ae3964ded9c7627848b:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T14:26:58,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:26:58,376 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:26:58,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fa04d9e7f6ce6ae3964ded9c7627848b:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T14:26:58,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:26:58,380 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:26:58,380 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1541): fa04d9e7f6ce6ae3964ded9c7627848b/B is initiating minor compaction (all files) 2024-11-23T14:26:58,380 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fa04d9e7f6ce6ae3964ded9c7627848b/B in TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:58,380 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/394c8fcd86f546fa9c6ee1440f209d0e, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/3ddbcb1a226d449197f0ae5962faf468, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/3c3a0a9c77344505b66abe8a76b31219] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp, totalSize=36.8 K 2024-11-23T14:26:58,381 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 394c8fcd86f546fa9c6ee1440f209d0e, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1732372014729 2024-11-23T14:26:58,381 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ddbcb1a226d449197f0ae5962faf468, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1732372015450 2024-11-23T14:26:58,381 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c3a0a9c77344505b66abe8a76b31219, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1732372016143 2024-11-23T14:26:58,383 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fa04d9e7f6ce6ae3964ded9c7627848b] 2024-11-23T14:26:58,397 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241123f8cdb76ffc88402abc0b19c80c635f22_fa04d9e7f6ce6ae3964ded9c7627848b store=[table=TestAcidGuarantees family=A region=fa04d9e7f6ce6ae3964ded9c7627848b] 2024-11-23T14:26:58,402 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241123f8cdb76ffc88402abc0b19c80c635f22_fa04d9e7f6ce6ae3964ded9c7627848b, store=[table=TestAcidGuarantees family=A region=fa04d9e7f6ce6ae3964ded9c7627848b] 2024-11-23T14:26:58,402 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123f8cdb76ffc88402abc0b19c80c635f22_fa04d9e7f6ce6ae3964ded9c7627848b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fa04d9e7f6ce6ae3964ded9c7627848b] 2024-11-23T14:26:58,410 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-23T14:26:58,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:58,411 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing fa04d9e7f6ce6ae3964ded9c7627848b 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-23T14:26:58,413 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fa04d9e7f6ce6ae3964ded9c7627848b#B#compaction#390 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:26:58,413 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/B/8d08e26db09f4251878128b2bb8b42bf is 50, key is test_row_0/B:col10/1732372017315/Put/seqid=0 2024-11-23T14:26:58,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742286_1462 (size=4469) 2024-11-23T14:26:58,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411233c7e0a603335474098de9e3eb98ccfc9_fa04d9e7f6ce6ae3964ded9c7627848b is 50, key is test_row_0/A:col10/1732372017354/Put/seqid=0 2024-11-23T14:26:58,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742287_1463 (size=13153) 2024-11-23T14:26:58,499 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/B/8d08e26db09f4251878128b2bb8b42bf as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/8d08e26db09f4251878128b2bb8b42bf 2024-11-23T14:26:58,510 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fa04d9e7f6ce6ae3964ded9c7627848b/B of fa04d9e7f6ce6ae3964ded9c7627848b into 8d08e26db09f4251878128b2bb8b42bf(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:26:58,510 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fa04d9e7f6ce6ae3964ded9c7627848b: 2024-11-23T14:26:58,510 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., storeName=fa04d9e7f6ce6ae3964ded9c7627848b/B, priority=13, startTime=1732372018376; duration=0sec 2024-11-23T14:26:58,510 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:26:58,510 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fa04d9e7f6ce6ae3964ded9c7627848b:B 2024-11-23T14:26:58,510 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:26:58,511 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:26:58,511 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1541): fa04d9e7f6ce6ae3964ded9c7627848b/C is initiating minor compaction (all files) 2024-11-23T14:26:58,511 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fa04d9e7f6ce6ae3964ded9c7627848b/C in TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:58,512 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/1be829ff64c74c919b1781ea5e873d07, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/5fec32ffb0ba41ef8b17f29acc847807, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/a7e21b5ba2834ddca414c9b17bf1a105] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp, totalSize=36.7 K 2024-11-23T14:26:58,512 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 1be829ff64c74c919b1781ea5e873d07, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1732372014729 2024-11-23T14:26:58,512 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 5fec32ffb0ba41ef8b17f29acc847807, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1732372015450 2024-11-23T14:26:58,514 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting a7e21b5ba2834ddca414c9b17bf1a105, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1732372016143 2024-11-23T14:26:58,533 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fa04d9e7f6ce6ae3964ded9c7627848b#C#compaction#392 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:26:58,534 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/C/c21fdccb6eca49ac9f47b4583e089c8a is 50, key is test_row_0/C:col10/1732372017315/Put/seqid=0 2024-11-23T14:26:58,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742288_1464 (size=12454) 2024-11-23T14:26:58,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on fa04d9e7f6ce6ae3964ded9c7627848b 2024-11-23T14:26:58,546 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. as already flushing 2024-11-23T14:26:58,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742289_1465 (size=13119) 2024-11-23T14:26:58,615 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:58,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32856 deadline: 1732372078598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:58,616 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:58,616 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:58,616 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:58,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:58,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32838 deadline: 1732372078600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:58,616 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:58,616 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:58,616 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:58,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-23T14:26:58,621 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:58,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32848 deadline: 1732372078603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:58,621 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:58,621 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:58,621 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:58,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:58,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32882 deadline: 1732372078616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:58,629 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:58,629 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:58,629 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:58,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:58,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32896 deadline: 1732372078616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:58,632 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:58,632 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:58,632 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:58,726 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:58,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32838 deadline: 1732372078721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:58,727 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:58,727 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:58,727 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:58,727 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:58,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32856 deadline: 1732372078721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:58,727 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:58,727 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:58,727 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:58,743 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:58,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32848 deadline: 1732372078730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:58,744 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:58,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:58,745 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:58,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:58,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32882 deadline: 1732372078740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:58,748 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:58,748 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:58,748 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:58,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:58,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32896 deadline: 1732372078740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:58,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:58,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:58,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:58,887 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fa04d9e7f6ce6ae3964ded9c7627848b#A#compaction#389 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:26:58,887 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/A/2dc07326ed48407dbfac54605b600c0a is 175, key is test_row_0/A:col10/1732372017315/Put/seqid=0 2024-11-23T14:26:58,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742290_1466 (size=32107) 2024-11-23T14:26:58,914 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/A/2dc07326ed48407dbfac54605b600c0a as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/2dc07326ed48407dbfac54605b600c0a 2024-11-23T14:26:58,920 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fa04d9e7f6ce6ae3964ded9c7627848b/A of fa04d9e7f6ce6ae3964ded9c7627848b into 2dc07326ed48407dbfac54605b600c0a(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:26:58,921 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fa04d9e7f6ce6ae3964ded9c7627848b: 2024-11-23T14:26:58,921 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., storeName=fa04d9e7f6ce6ae3964ded9c7627848b/A, priority=13, startTime=1732372018366; duration=0sec 2024-11-23T14:26:58,921 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:26:58,921 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fa04d9e7f6ce6ae3964ded9c7627848b:A 2024-11-23T14:26:58,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:26:58,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:58,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32838 deadline: 1732372078930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:58,941 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:58,941 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:58,941 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:58,942 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411233c7e0a603335474098de9e3eb98ccfc9_fa04d9e7f6ce6ae3964ded9c7627848b to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411233c7e0a603335474098de9e3eb98ccfc9_fa04d9e7f6ce6ae3964ded9c7627848b 2024-11-23T14:26:58,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/A/9b7bb6caa0bf4f11a2ba61ab40e25f1a, store: [table=TestAcidGuarantees family=A region=fa04d9e7f6ce6ae3964ded9c7627848b] 2024-11-23T14:26:58,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/A/9b7bb6caa0bf4f11a2ba61ab40e25f1a is 175, key is test_row_0/A:col10/1732372017354/Put/seqid=0 2024-11-23T14:26:58,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:58,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32856 deadline: 1732372078940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:58,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:58,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:58,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:58,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742291_1467 (size=31255) 2024-11-23T14:26:58,955 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=364, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/A/9b7bb6caa0bf4f11a2ba61ab40e25f1a 2024-11-23T14:26:58,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:58,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32848 deadline: 1732372078951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:58,962 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:58,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:58,962 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:58,962 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:58,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32882 deadline: 1732372078951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:58,963 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:58,963 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:58,963 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:58,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:58,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32896 deadline: 1732372078960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:58,972 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:58,972 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:58,972 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:58,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/B/b762c72a64514bdeb3aa101a5853d6d2 is 50, key is test_row_0/B:col10/1732372017354/Put/seqid=0 2024-11-23T14:26:58,995 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/C/c21fdccb6eca49ac9f47b4583e089c8a as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/c21fdccb6eca49ac9f47b4583e089c8a 2024-11-23T14:26:59,003 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fa04d9e7f6ce6ae3964ded9c7627848b/C of fa04d9e7f6ce6ae3964ded9c7627848b into c21fdccb6eca49ac9f47b4583e089c8a(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:26:59,003 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fa04d9e7f6ce6ae3964ded9c7627848b: 2024-11-23T14:26:59,003 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., storeName=fa04d9e7f6ce6ae3964ded9c7627848b/C, priority=13, startTime=1732372018376; duration=0sec 2024-11-23T14:26:59,003 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:26:59,003 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fa04d9e7f6ce6ae3964ded9c7627848b:C 2024-11-23T14:26:59,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742292_1468 (size=12301) 2024-11-23T14:26:59,008 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/B/b762c72a64514bdeb3aa101a5853d6d2 2024-11-23T14:26:59,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/C/5b05f4829a23462199da18a78d0c444c is 50, key is test_row_0/C:col10/1732372017354/Put/seqid=0 2024-11-23T14:26:59,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742293_1469 (size=12301) 2024-11-23T14:26:59,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:59,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32838 deadline: 1732372079251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:59,260 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:59,260 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:59,260 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:59,264 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:59,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32856 deadline: 1732372079261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:59,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:59,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:59,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:59,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:59,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32882 deadline: 1732372079270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:59,275 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:59,275 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:59,276 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:59,279 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:59,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32848 deadline: 1732372079272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:59,280 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:59,280 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:59,280 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:59,290 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:59,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32896 deadline: 1732372079281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:59,291 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:59,291 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:59,291 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:59,430 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/C/5b05f4829a23462199da18a78d0c444c 2024-11-23T14:26:59,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/A/9b7bb6caa0bf4f11a2ba61ab40e25f1a as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/9b7bb6caa0bf4f11a2ba61ab40e25f1a 2024-11-23T14:26:59,439 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/9b7bb6caa0bf4f11a2ba61ab40e25f1a, entries=150, sequenceid=364, filesize=30.5 K 2024-11-23T14:26:59,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/B/b762c72a64514bdeb3aa101a5853d6d2 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/b762c72a64514bdeb3aa101a5853d6d2 2024-11-23T14:26:59,445 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/b762c72a64514bdeb3aa101a5853d6d2, entries=150, sequenceid=364, filesize=12.0 K 2024-11-23T14:26:59,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/C/5b05f4829a23462199da18a78d0c444c as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/5b05f4829a23462199da18a78d0c444c 2024-11-23T14:26:59,451 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/5b05f4829a23462199da18a78d0c444c, entries=150, sequenceid=364, filesize=12.0 K 2024-11-23T14:26:59,452 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for fa04d9e7f6ce6ae3964ded9c7627848b in 1042ms, sequenceid=364, compaction requested=false 2024-11-23T14:26:59,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for fa04d9e7f6ce6ae3964ded9c7627848b: 2024-11-23T14:26:59,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:59,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-11-23T14:26:59,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-11-23T14:26:59,456 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-23T14:26:59,456 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9700 sec 2024-11-23T14:26:59,458 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 1.9760 sec 2024-11-23T14:26:59,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-23T14:26:59,631 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T14:26:59,631 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-23T14:26:59,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-11-23T14:26:59,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-23T14:26:59,633 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T14:26:59,634 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T14:26:59,634 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T14:26:59,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-23T14:26:59,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on fa04d9e7f6ce6ae3964ded9c7627848b 2024-11-23T14:26:59,773 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T14:26:59,774 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fa04d9e7f6ce6ae3964ded9c7627848b 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-23T14:26:59,786 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-23T14:26:59,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:59,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. as already flushing 2024-11-23T14:26:59,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:59,787 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:59,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:59,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:59,792 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123285a9d7db40c4dd59900fd55df525df0_fa04d9e7f6ce6ae3964ded9c7627848b is 50, key is test_row_0/A:col10/1732372018588/Put/seqid=0 2024-11-23T14:26:59,826 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:59,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32882 deadline: 1732372079813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:59,826 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:59,826 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:59,826 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:59,830 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:59,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32848 deadline: 1732372079815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:59,830 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:59,830 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:59,830 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:59,830 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:59,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32896 deadline: 1732372079817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:59,830 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:59,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:59,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:59,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742294_1470 (size=14994) 2024-11-23T14:26:59,838 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:26:59,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:59,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32856 deadline: 1732372079826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:59,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:59,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:59,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:59,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:59,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32838 deadline: 1732372079827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:59,842 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:59,842 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:59,842 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:59,844 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123285a9d7db40c4dd59900fd55df525df0_fa04d9e7f6ce6ae3964ded9c7627848b to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123285a9d7db40c4dd59900fd55df525df0_fa04d9e7f6ce6ae3964ded9c7627848b 2024-11-23T14:26:59,845 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/A/e2b408beb88a466bb0d6d6157a7c65bc, store: [table=TestAcidGuarantees family=A region=fa04d9e7f6ce6ae3964ded9c7627848b] 2024-11-23T14:26:59,846 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/A/e2b408beb88a466bb0d6d6157a7c65bc is 175, key is test_row_0/A:col10/1732372018588/Put/seqid=0 2024-11-23T14:26:59,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742295_1471 (size=39949) 2024-11-23T14:26:59,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:59,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32882 deadline: 1732372079930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:59,936 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:59,936 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:59,936 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:59,943 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-23T14:26:59,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:59,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. as already flushing 2024-11-23T14:26:59,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:26:59,943 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:59,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:59,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:26:59,948 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:59,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32848 deadline: 1732372079941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:59,948 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:59,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32896 deadline: 1732372079941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:59,949 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:59,949 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:59,949 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:59,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-23T14:26:59,950 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:59,950 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:59,950 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:59,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:59,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32856 deadline: 1732372079950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:59,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:26:59,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32838 deadline: 1732372079950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:26:59,959 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:59,959 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:59,959 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:26:59,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:59,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:26:59,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:00,098 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-23T14:27:00,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:27:00,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. as already flushing 2024-11-23T14:27:00,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:27:00,098 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:00,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:00,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:00,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:00,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32882 deadline: 1732372080140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:00,147 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:00,147 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:00,147 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:00,169 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:00,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32896 deadline: 1732372080160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:00,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:00,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32848 deadline: 1732372080160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:00,170 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:00,170 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:00,170 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:00,170 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:00,171 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:00,171 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:00,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:00,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32838 deadline: 1732372080170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:00,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:00,178 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:00,178 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:00,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32856 deadline: 1732372080171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:00,178 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:00,178 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:00,178 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:00,178 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:00,251 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-23T14:27:00,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:27:00,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. as already flushing 2024-11-23T14:27:00,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:27:00,251 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:00,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:00,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:00,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-23T14:27:00,290 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=387, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/A/e2b408beb88a466bb0d6d6157a7c65bc 2024-11-23T14:27:00,306 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/B/3aad82271d474a698f809a27acd4afd6 is 50, key is test_row_0/B:col10/1732372018588/Put/seqid=0 2024-11-23T14:27:00,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742296_1472 (size=12301) 2024-11-23T14:27:00,354 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=387 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/B/3aad82271d474a698f809a27acd4afd6 2024-11-23T14:27:00,375 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/C/4d3b9508600c481fb25d4f0e1b833040 is 50, key is test_row_0/C:col10/1732372018588/Put/seqid=0 2024-11-23T14:27:00,404 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-23T14:27:00,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:27:00,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. as already flushing 2024-11-23T14:27:00,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:27:00,404 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:00,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:00,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:00,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742297_1473 (size=12301) 2024-11-23T14:27:00,463 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:00,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32882 deadline: 1732372080460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:00,464 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:00,464 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:00,464 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:00,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:00,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32848 deadline: 1732372080480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:00,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:00,488 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:00,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32896 deadline: 1732372080480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:00,488 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:00,489 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:00,489 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:00,489 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:00,489 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:00,492 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:00,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32856 deadline: 1732372080490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:00,493 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:00,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32838 deadline: 1732372080490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:00,493 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:00,493 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:00,493 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:00,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:00,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:00,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:00,558 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-23T14:27:00,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:27:00,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. as already flushing 2024-11-23T14:27:00,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:27:00,558 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:00,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:00,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:00,713 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-23T14:27:00,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:27:00,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. as already flushing 2024-11-23T14:27:00,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:27:00,713 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:00,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:00,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:00,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-23T14:27:00,797 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-23T14:27:00,810 INFO [Thread-1652 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1652. 2024-11-23T14:27:00,810 DEBUG [Thread-1652 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T14:27:00,810 DEBUG [Thread-1652 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:27:00,810 DEBUG [Thread-1652 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:27:00,810 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T14:27:00,814 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=387 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/C/4d3b9508600c481fb25d4f0e1b833040 2024-11-23T14:27:00,815 INFO [Thread-1650 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1650. 2024-11-23T14:27:00,815 DEBUG [Thread-1650 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T14:27:00,815 DEBUG [Thread-1650 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:27:00,815 DEBUG [Thread-1650 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:27:00,815 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T14:27:00,817 INFO [Thread-1656 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1656. 2024-11-23T14:27:00,817 DEBUG [Thread-1656 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T14:27:00,817 DEBUG [Thread-1656 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:27:00,817 DEBUG [Thread-1656 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:27:00,817 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T14:27:00,819 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/A/e2b408beb88a466bb0d6d6157a7c65bc as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/e2b408beb88a466bb0d6d6157a7c65bc 2024-11-23T14:27:00,822 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/e2b408beb88a466bb0d6d6157a7c65bc, entries=200, sequenceid=387, filesize=39.0 K 2024-11-23T14:27:00,822 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/B/3aad82271d474a698f809a27acd4afd6 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/3aad82271d474a698f809a27acd4afd6 2024-11-23T14:27:00,823 INFO [Thread-1654 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1654. 2024-11-23T14:27:00,823 DEBUG [Thread-1654 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T14:27:00,823 DEBUG [Thread-1654 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:27:00,823 DEBUG [Thread-1654 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:27:00,823 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T14:27:00,826 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/3aad82271d474a698f809a27acd4afd6, entries=150, sequenceid=387, filesize=12.0 K 2024-11-23T14:27:00,828 INFO [Thread-1658 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1658. 2024-11-23T14:27:00,828 DEBUG [Thread-1658 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T14:27:00,828 DEBUG [Thread-1658 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:27:00,828 DEBUG [Thread-1658 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:27:00,828 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/C/4d3b9508600c481fb25d4f0e1b833040 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/4d3b9508600c481fb25d4f0e1b833040 2024-11-23T14:27:00,828 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T14:27:00,831 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/4d3b9508600c481fb25d4f0e1b833040, entries=150, sequenceid=387, filesize=12.0 K 2024-11-23T14:27:00,832 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=93.93 KB/96180 for fa04d9e7f6ce6ae3964ded9c7627848b in 1058ms, sequenceid=387, compaction requested=true 2024-11-23T14:27:00,832 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fa04d9e7f6ce6ae3964ded9c7627848b: 2024-11-23T14:27:00,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fa04d9e7f6ce6ae3964ded9c7627848b:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T14:27:00,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:27:00,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fa04d9e7f6ce6ae3964ded9c7627848b:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T14:27:00,833 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:27:00,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:27:00,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fa04d9e7f6ce6ae3964ded9c7627848b:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T14:27:00,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:27:00,833 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:27:00,833 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:27:00,833 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103311 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:27:00,833 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1541): fa04d9e7f6ce6ae3964ded9c7627848b/B is initiating minor compaction (all files) 2024-11-23T14:27:00,833 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1541): fa04d9e7f6ce6ae3964ded9c7627848b/A is initiating minor compaction (all files) 2024-11-23T14:27:00,833 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fa04d9e7f6ce6ae3964ded9c7627848b/B in TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:27:00,833 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fa04d9e7f6ce6ae3964ded9c7627848b/A in TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:27:00,834 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/2dc07326ed48407dbfac54605b600c0a, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/9b7bb6caa0bf4f11a2ba61ab40e25f1a, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/e2b408beb88a466bb0d6d6157a7c65bc] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp, totalSize=100.9 K 2024-11-23T14:27:00,834 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/8d08e26db09f4251878128b2bb8b42bf, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/b762c72a64514bdeb3aa101a5853d6d2, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/3aad82271d474a698f809a27acd4afd6] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp, totalSize=36.9 K 2024-11-23T14:27:00,834 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:27:00,834 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. files: [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/2dc07326ed48407dbfac54605b600c0a, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/9b7bb6caa0bf4f11a2ba61ab40e25f1a, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/e2b408beb88a466bb0d6d6157a7c65bc] 2024-11-23T14:27:00,834 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d08e26db09f4251878128b2bb8b42bf, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1732372016143 2024-11-23T14:27:00,834 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2dc07326ed48407dbfac54605b600c0a, keycount=150, bloomtype=ROW, size=31.4 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1732372016143 2024-11-23T14:27:00,834 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting b762c72a64514bdeb3aa101a5853d6d2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1732372017354 2024-11-23T14:27:00,834 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b7bb6caa0bf4f11a2ba61ab40e25f1a, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1732372017354 2024-11-23T14:27:00,834 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 3aad82271d474a698f809a27acd4afd6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1732372018588 2024-11-23T14:27:00,834 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting e2b408beb88a466bb0d6d6157a7c65bc, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1732372018588 2024-11-23T14:27:00,839 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fa04d9e7f6ce6ae3964ded9c7627848b] 2024-11-23T14:27:00,841 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112342cde448871b4823b3407185b371a5ab_fa04d9e7f6ce6ae3964ded9c7627848b store=[table=TestAcidGuarantees family=A region=fa04d9e7f6ce6ae3964ded9c7627848b] 2024-11-23T14:27:00,842 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fa04d9e7f6ce6ae3964ded9c7627848b#B#compaction#399 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:27:00,842 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/B/c8b627902d0a4ae2bdac5717a132430f is 50, key is test_row_0/B:col10/1732372018588/Put/seqid=0 2024-11-23T14:27:00,843 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112342cde448871b4823b3407185b371a5ab_fa04d9e7f6ce6ae3964ded9c7627848b, store=[table=TestAcidGuarantees family=A region=fa04d9e7f6ce6ae3964ded9c7627848b] 2024-11-23T14:27:00,843 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112342cde448871b4823b3407185b371a5ab_fa04d9e7f6ce6ae3964ded9c7627848b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fa04d9e7f6ce6ae3964ded9c7627848b] 2024-11-23T14:27:00,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742299_1475 (size=4469) 2024-11-23T14:27:00,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742298_1474 (size=13255) 2024-11-23T14:27:00,866 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/B/c8b627902d0a4ae2bdac5717a132430f as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/c8b627902d0a4ae2bdac5717a132430f 2024-11-23T14:27:00,867 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-23T14:27:00,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:27:00,867 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing fa04d9e7f6ce6ae3964ded9c7627848b 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-23T14:27:00,873 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fa04d9e7f6ce6ae3964ded9c7627848b/B of fa04d9e7f6ce6ae3964ded9c7627848b into c8b627902d0a4ae2bdac5717a132430f(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:27:00,873 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fa04d9e7f6ce6ae3964ded9c7627848b: 2024-11-23T14:27:00,873 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., storeName=fa04d9e7f6ce6ae3964ded9c7627848b/B, priority=13, startTime=1732372020833; duration=0sec 2024-11-23T14:27:00,873 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:27:00,873 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fa04d9e7f6ce6ae3964ded9c7627848b:B 2024-11-23T14:27:00,873 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:27:00,879 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:27:00,879 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1541): fa04d9e7f6ce6ae3964ded9c7627848b/C is initiating minor compaction (all files) 2024-11-23T14:27:00,879 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fa04d9e7f6ce6ae3964ded9c7627848b/C in TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:27:00,879 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/c21fdccb6eca49ac9f47b4583e089c8a, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/5b05f4829a23462199da18a78d0c444c, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/4d3b9508600c481fb25d4f0e1b833040] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp, totalSize=36.8 K 2024-11-23T14:27:00,880 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting c21fdccb6eca49ac9f47b4583e089c8a, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1732372016143 2024-11-23T14:27:00,880 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b05f4829a23462199da18a78d0c444c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1732372017354 2024-11-23T14:27:00,880 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d3b9508600c481fb25d4f0e1b833040, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1732372018588 2024-11-23T14:27:00,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411239345fc7e3f0346c1b478517cddd89e5c_fa04d9e7f6ce6ae3964ded9c7627848b is 50, key is test_row_0/A:col10/1732372019815/Put/seqid=0 2024-11-23T14:27:00,894 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fa04d9e7f6ce6ae3964ded9c7627848b#C#compaction#401 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:27:00,894 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/C/0908cde42e4c42f3810ed1f30dd1ac2c is 50, key is test_row_0/C:col10/1732372018588/Put/seqid=0 2024-11-23T14:27:00,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742300_1476 (size=12454) 2024-11-23T14:27:00,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742301_1477 (size=13221) 2024-11-23T14:27:00,915 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/C/0908cde42e4c42f3810ed1f30dd1ac2c as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/0908cde42e4c42f3810ed1f30dd1ac2c 2024-11-23T14:27:00,919 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fa04d9e7f6ce6ae3964ded9c7627848b/C of fa04d9e7f6ce6ae3964ded9c7627848b into 0908cde42e4c42f3810ed1f30dd1ac2c(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:27:00,919 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fa04d9e7f6ce6ae3964ded9c7627848b: 2024-11-23T14:27:00,919 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., storeName=fa04d9e7f6ce6ae3964ded9c7627848b/C, priority=13, startTime=1732372020833; duration=0sec 2024-11-23T14:27:00,919 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:27:00,919 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fa04d9e7f6ce6ae3964ded9c7627848b:C 2024-11-23T14:27:00,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on fa04d9e7f6ce6ae3964ded9c7627848b 2024-11-23T14:27:00,971 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. as already flushing 2024-11-23T14:27:00,971 INFO [Thread-1647 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1647. 2024-11-23T14:27:00,971 DEBUG [Thread-1647 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T14:27:00,971 DEBUG [Thread-1647 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:27:00,971 DEBUG [Thread-1647 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:27:00,971 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T14:27:01,002 INFO [Thread-1643 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1643. 2024-11-23T14:27:01,002 INFO [Thread-1645 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1645. 2024-11-23T14:27:01,002 DEBUG [Thread-1645 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T14:27:01,002 DEBUG [Thread-1643 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T14:27:01,003 DEBUG [Thread-1645 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:27:01,003 DEBUG [Thread-1643 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:27:01,003 DEBUG [Thread-1645 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:27:01,003 DEBUG [Thread-1643 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:27:01,003 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T14:27:01,003 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T14:27:01,003 INFO [Thread-1641 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1641. 2024-11-23T14:27:01,003 DEBUG [Thread-1641 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T14:27:01,003 DEBUG [Thread-1641 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:27:01,003 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T14:27:01,004 DEBUG [Thread-1641 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:27:01,005 INFO [Thread-1639 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1639. 2024-11-23T14:27:01,005 DEBUG [Thread-1639 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T14:27:01,005 DEBUG [Thread-1639 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:27:01,006 DEBUG [Thread-1639 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:27:01,006 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T14:27:01,255 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fa04d9e7f6ce6ae3964ded9c7627848b#A#compaction#398 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:27:01,256 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/A/520dbaeddf0c417c8124c1a615edfdc2 is 175, key is test_row_0/A:col10/1732372018588/Put/seqid=0 2024-11-23T14:27:01,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742302_1478 (size=32209) 2024-11-23T14:27:01,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:01,306 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411239345fc7e3f0346c1b478517cddd89e5c_fa04d9e7f6ce6ae3964ded9c7627848b to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411239345fc7e3f0346c1b478517cddd89e5c_fa04d9e7f6ce6ae3964ded9c7627848b 2024-11-23T14:27:01,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/A/5a13cdb2806e4b9a9456d81ba42a4838, store: [table=TestAcidGuarantees family=A region=fa04d9e7f6ce6ae3964ded9c7627848b] 2024-11-23T14:27:01,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/A/5a13cdb2806e4b9a9456d81ba42a4838 is 175, key is test_row_0/A:col10/1732372019815/Put/seqid=0 2024-11-23T14:27:01,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742303_1479 (size=31255) 2024-11-23T14:27:01,664 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/A/520dbaeddf0c417c8124c1a615edfdc2 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/520dbaeddf0c417c8124c1a615edfdc2 2024-11-23T14:27:01,671 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fa04d9e7f6ce6ae3964ded9c7627848b/A of fa04d9e7f6ce6ae3964ded9c7627848b into 520dbaeddf0c417c8124c1a615edfdc2(size=31.5 K), total size for store is 31.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:27:01,671 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fa04d9e7f6ce6ae3964ded9c7627848b: 2024-11-23T14:27:01,671 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., storeName=fa04d9e7f6ce6ae3964ded9c7627848b/A, priority=13, startTime=1732372020832; duration=0sec 2024-11-23T14:27:01,671 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:27:01,671 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fa04d9e7f6ce6ae3964ded9c7627848b:A 2024-11-23T14:27:01,714 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=404, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/A/5a13cdb2806e4b9a9456d81ba42a4838 2024-11-23T14:27:01,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/B/f61514ed1bef4550b0cbbb76ea91d473 is 50, key is test_row_0/B:col10/1732372019815/Put/seqid=0 2024-11-23T14:27:01,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742304_1480 (size=12301) 2024-11-23T14:27:01,730 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=404 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/B/f61514ed1bef4550b0cbbb76ea91d473 2024-11-23T14:27:01,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/C/162b02b59a5547bc882a93a75bf93f0e is 50, key is test_row_0/C:col10/1732372019815/Put/seqid=0 2024-11-23T14:27:01,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742305_1481 (size=12301) 2024-11-23T14:27:01,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-23T14:27:02,150 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=404 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/C/162b02b59a5547bc882a93a75bf93f0e 2024-11-23T14:27:02,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/A/5a13cdb2806e4b9a9456d81ba42a4838 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/5a13cdb2806e4b9a9456d81ba42a4838 2024-11-23T14:27:02,157 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/5a13cdb2806e4b9a9456d81ba42a4838, entries=150, sequenceid=404, filesize=30.5 K 2024-11-23T14:27:02,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/B/f61514ed1bef4550b0cbbb76ea91d473 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/f61514ed1bef4550b0cbbb76ea91d473 2024-11-23T14:27:02,160 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/f61514ed1bef4550b0cbbb76ea91d473, entries=150, sequenceid=404, filesize=12.0 K 2024-11-23T14:27:02,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/C/162b02b59a5547bc882a93a75bf93f0e as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/162b02b59a5547bc882a93a75bf93f0e 2024-11-23T14:27:02,163 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/162b02b59a5547bc882a93a75bf93f0e, entries=150, sequenceid=404, filesize=12.0 K 2024-11-23T14:27:02,164 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=33.54 KB/34350 for fa04d9e7f6ce6ae3964ded9c7627848b in 1297ms, sequenceid=404, compaction requested=false 2024-11-23T14:27:02,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for fa04d9e7f6ce6ae3964ded9c7627848b: 2024-11-23T14:27:02,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:27:02,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-11-23T14:27:02,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-11-23T14:27:02,166 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-23T14:27:02,166 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5310 sec 2024-11-23T14:27:02,168 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 2.5360 sec 2024-11-23T14:27:03,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-23T14:27:03,800 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T14:27:03,801 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-23T14:27:03,801 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 68 2024-11-23T14:27:03,801 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 60 2024-11-23T14:27:03,801 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 69 2024-11-23T14:27:03,801 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 58 2024-11-23T14:27:03,801 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 64 2024-11-23T14:27:03,801 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-23T14:27:03,801 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-23T14:27:03,801 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1289 2024-11-23T14:27:03,801 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3859 rows 2024-11-23T14:27:03,801 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1288 2024-11-23T14:27:03,801 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3860 rows 2024-11-23T14:27:03,801 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1294 2024-11-23T14:27:03,801 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3878 rows 2024-11-23T14:27:03,801 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1281 2024-11-23T14:27:03,801 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3837 rows 2024-11-23T14:27:03,801 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1276 2024-11-23T14:27:03,801 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3823 rows 2024-11-23T14:27:03,801 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-23T14:27:03,801 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T14:27:03,801 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:27:03,801 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:27:03,801 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T14:27:03,803 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-23T14:27:03,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-23T14:27:03,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-23T14:27:03,808 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732372023808"}]},"ts":"1732372023808"} 2024-11-23T14:27:03,810 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-23T14:27:03,810 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-23T14:27:03,810 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-23T14:27:03,812 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fa04d9e7f6ce6ae3964ded9c7627848b, UNASSIGN}] 2024-11-23T14:27:03,814 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fa04d9e7f6ce6ae3964ded9c7627848b, UNASSIGN 2024-11-23T14:27:03,817 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=121 updating hbase:meta row=fa04d9e7f6ce6ae3964ded9c7627848b, regionState=CLOSING, regionLocation=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:03,821 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fa04d9e7f6ce6ae3964ded9c7627848b, UNASSIGN because future has completed 2024-11-23T14:27:03,821 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-23T14:27:03,821 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; CloseRegionProcedure fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767}] 2024-11-23T14:27:03,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-23T14:27:03,975 INFO [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] handler.UnassignRegionHandler(122): Close fa04d9e7f6ce6ae3964ded9c7627848b 2024-11-23T14:27:03,975 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-23T14:27:03,975 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1722): Closing fa04d9e7f6ce6ae3964ded9c7627848b, disabling compactions & flushes 2024-11-23T14:27:03,975 INFO [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:27:03,975 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:27:03,975 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. after waiting 0 ms 2024-11-23T14:27:03,975 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:27:03,975 INFO [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(2902): Flushing fa04d9e7f6ce6ae3964ded9c7627848b 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-23T14:27:03,992 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123df5c33f0f28d470098935323957df4c9_fa04d9e7f6ce6ae3964ded9c7627848b is 50, key is test_row_0/A:col10/1732372021002/Put/seqid=0 2024-11-23T14:27:04,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742306_1482 (size=9914) 2024-11-23T14:27:04,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-23T14:27:04,406 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:04,409 INFO [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123df5c33f0f28d470098935323957df4c9_fa04d9e7f6ce6ae3964ded9c7627848b to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123df5c33f0f28d470098935323957df4c9_fa04d9e7f6ce6ae3964ded9c7627848b 2024-11-23T14:27:04,410 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/A/6ab6a8a00d70472e929bff2193866cc6, store: [table=TestAcidGuarantees family=A region=fa04d9e7f6ce6ae3964ded9c7627848b] 2024-11-23T14:27:04,411 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/A/6ab6a8a00d70472e929bff2193866cc6 is 175, key is test_row_0/A:col10/1732372021002/Put/seqid=0 2024-11-23T14:27:04,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742307_1483 (size=22561) 2024-11-23T14:27:04,415 INFO [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=415, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/A/6ab6a8a00d70472e929bff2193866cc6 2024-11-23T14:27:04,421 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/B/a6b6118637104b81a263f0b23eb44f61 is 50, key is test_row_0/B:col10/1732372021002/Put/seqid=0 2024-11-23T14:27:04,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742308_1484 (size=9857) 2024-11-23T14:27:04,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-23T14:27:04,824 INFO [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/B/a6b6118637104b81a263f0b23eb44f61 2024-11-23T14:27:04,830 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/C/347bcc25b773463ca88e2e4c22af2912 is 50, key is test_row_0/C:col10/1732372021002/Put/seqid=0 2024-11-23T14:27:04,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742309_1485 (size=9857) 2024-11-23T14:27:04,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-23T14:27:05,235 INFO [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/C/347bcc25b773463ca88e2e4c22af2912 2024-11-23T14:27:05,239 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/A/6ab6a8a00d70472e929bff2193866cc6 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/6ab6a8a00d70472e929bff2193866cc6 2024-11-23T14:27:05,242 INFO [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/6ab6a8a00d70472e929bff2193866cc6, entries=100, sequenceid=415, filesize=22.0 K 2024-11-23T14:27:05,242 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/B/a6b6118637104b81a263f0b23eb44f61 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/a6b6118637104b81a263f0b23eb44f61 2024-11-23T14:27:05,245 INFO [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/a6b6118637104b81a263f0b23eb44f61, entries=100, sequenceid=415, filesize=9.6 K 2024-11-23T14:27:05,246 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/.tmp/C/347bcc25b773463ca88e2e4c22af2912 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/347bcc25b773463ca88e2e4c22af2912 2024-11-23T14:27:05,249 INFO [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/347bcc25b773463ca88e2e4c22af2912, entries=100, sequenceid=415, filesize=9.6 K 2024-11-23T14:27:05,250 INFO [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for fa04d9e7f6ce6ae3964ded9c7627848b in 1275ms, sequenceid=415, compaction requested=true 2024-11-23T14:27:05,250 DEBUG [StoreCloser-TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/389f3cb1fc5a4c20b1deef0d5473f6cb, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/8344cf9e6c7746d38920b367666524f9, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/a3909864813b4763b7c286f25b69d532, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/85e7b9427e124a6e93e8d5e1b7787e2f, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/8e9828f390ed48c18def9091b530da6b, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/ffc54d5f6cd54cedb9c877da92d2e89b, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/615a4c27ae5d411987c99d2ca1199948, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/bff842b8b05b4f7eb17ed849e8a5f85b, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/7539e5610b0c455a8ae787d6fa8c2cfe, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/6449f3cc1b6a48d68fb958cf1fc864ba, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/ada406989bf94772a9b603e22f4d1d45, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/ef53055587144cea85be65ae94d248e4, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/51e49563a8584e148b64a7932bd1312f, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/391b52b702c646c1bee649b6411bc1f8, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/848550e06ab04428862d25b3b5cdbebe, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/aebc6545cc5d4b16af8456cfa25c7272, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/936fc6eca3424aaeb9dc7d75ec6f0400, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/a39f40eee3d449d9983773eb0ac23deb, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/a8b8b4248f70468c97b69bab16a8fe84, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/0a1bd3224fbd4c4d83d15d5d071cd781, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/cd45d1ce987743d380ac514180bcac0a, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/f0fa6dc573ef455cb66aaea113359951, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/6ed354cb0df5484391030953e63c5e13, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/647ecb3218404e4b846a6c6b1d6231e7, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/50d49f4c642a4f359ef7a5723de02d52, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/2dc07326ed48407dbfac54605b600c0a, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/9b7bb6caa0bf4f11a2ba61ab40e25f1a, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/e2b408beb88a466bb0d6d6157a7c65bc] to archive 2024-11-23T14:27:05,251 DEBUG [StoreCloser-TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-23T14:27:05,252 DEBUG [StoreCloser-TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/389f3cb1fc5a4c20b1deef0d5473f6cb to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/389f3cb1fc5a4c20b1deef0d5473f6cb 2024-11-23T14:27:05,253 DEBUG [StoreCloser-TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/8344cf9e6c7746d38920b367666524f9 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/8344cf9e6c7746d38920b367666524f9 2024-11-23T14:27:05,254 DEBUG [StoreCloser-TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/a3909864813b4763b7c286f25b69d532 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/a3909864813b4763b7c286f25b69d532 2024-11-23T14:27:05,255 DEBUG [StoreCloser-TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/85e7b9427e124a6e93e8d5e1b7787e2f to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/85e7b9427e124a6e93e8d5e1b7787e2f 2024-11-23T14:27:05,255 DEBUG [StoreCloser-TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/8e9828f390ed48c18def9091b530da6b to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/8e9828f390ed48c18def9091b530da6b 2024-11-23T14:27:05,256 DEBUG [StoreCloser-TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/ffc54d5f6cd54cedb9c877da92d2e89b to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/ffc54d5f6cd54cedb9c877da92d2e89b 2024-11-23T14:27:05,257 DEBUG [StoreCloser-TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/615a4c27ae5d411987c99d2ca1199948 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/615a4c27ae5d411987c99d2ca1199948 2024-11-23T14:27:05,258 DEBUG [StoreCloser-TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/bff842b8b05b4f7eb17ed849e8a5f85b to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/bff842b8b05b4f7eb17ed849e8a5f85b 2024-11-23T14:27:05,258 DEBUG [StoreCloser-TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/7539e5610b0c455a8ae787d6fa8c2cfe to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/7539e5610b0c455a8ae787d6fa8c2cfe 2024-11-23T14:27:05,259 DEBUG [StoreCloser-TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/6449f3cc1b6a48d68fb958cf1fc864ba to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/6449f3cc1b6a48d68fb958cf1fc864ba 2024-11-23T14:27:05,259 DEBUG [StoreCloser-TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/ada406989bf94772a9b603e22f4d1d45 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/ada406989bf94772a9b603e22f4d1d45 2024-11-23T14:27:05,260 DEBUG [StoreCloser-TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/ef53055587144cea85be65ae94d248e4 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/ef53055587144cea85be65ae94d248e4 2024-11-23T14:27:05,261 DEBUG [StoreCloser-TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/51e49563a8584e148b64a7932bd1312f to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/51e49563a8584e148b64a7932bd1312f 2024-11-23T14:27:05,261 DEBUG [StoreCloser-TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/391b52b702c646c1bee649b6411bc1f8 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/391b52b702c646c1bee649b6411bc1f8 2024-11-23T14:27:05,262 DEBUG [StoreCloser-TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/848550e06ab04428862d25b3b5cdbebe to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/848550e06ab04428862d25b3b5cdbebe 2024-11-23T14:27:05,263 DEBUG [StoreCloser-TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/aebc6545cc5d4b16af8456cfa25c7272 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/aebc6545cc5d4b16af8456cfa25c7272 2024-11-23T14:27:05,263 DEBUG [StoreCloser-TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/936fc6eca3424aaeb9dc7d75ec6f0400 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/936fc6eca3424aaeb9dc7d75ec6f0400 2024-11-23T14:27:05,264 DEBUG [StoreCloser-TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/a39f40eee3d449d9983773eb0ac23deb to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/a39f40eee3d449d9983773eb0ac23deb 2024-11-23T14:27:05,265 DEBUG [StoreCloser-TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/a8b8b4248f70468c97b69bab16a8fe84 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/a8b8b4248f70468c97b69bab16a8fe84 2024-11-23T14:27:05,265 DEBUG [StoreCloser-TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/0a1bd3224fbd4c4d83d15d5d071cd781 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/0a1bd3224fbd4c4d83d15d5d071cd781 2024-11-23T14:27:05,266 DEBUG [StoreCloser-TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/cd45d1ce987743d380ac514180bcac0a to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/cd45d1ce987743d380ac514180bcac0a 2024-11-23T14:27:05,267 DEBUG [StoreCloser-TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/f0fa6dc573ef455cb66aaea113359951 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/f0fa6dc573ef455cb66aaea113359951 2024-11-23T14:27:05,268 DEBUG [StoreCloser-TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/6ed354cb0df5484391030953e63c5e13 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/6ed354cb0df5484391030953e63c5e13 2024-11-23T14:27:05,269 DEBUG [StoreCloser-TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/647ecb3218404e4b846a6c6b1d6231e7 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/647ecb3218404e4b846a6c6b1d6231e7 2024-11-23T14:27:05,269 DEBUG [StoreCloser-TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/50d49f4c642a4f359ef7a5723de02d52 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/50d49f4c642a4f359ef7a5723de02d52 2024-11-23T14:27:05,270 DEBUG [StoreCloser-TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/2dc07326ed48407dbfac54605b600c0a to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/2dc07326ed48407dbfac54605b600c0a 2024-11-23T14:27:05,271 DEBUG [StoreCloser-TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/9b7bb6caa0bf4f11a2ba61ab40e25f1a to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/9b7bb6caa0bf4f11a2ba61ab40e25f1a 2024-11-23T14:27:05,271 DEBUG [StoreCloser-TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/e2b408beb88a466bb0d6d6157a7c65bc to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/e2b408beb88a466bb0d6d6157a7c65bc 2024-11-23T14:27:05,274 DEBUG [StoreCloser-TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/fc69506ab2b048cf9019543873e0a017, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/b77d2da995604fd59e03f54dd9ed9297, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/a897475d1eea46c29c3ca2cd1c3bc7be, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/cc893cfd2fb548b3a42a1047b872a8c3, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/f4500ed0cbc94b57aefafb5e08e6f5a2, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/419ea993d355482ab74361efef7ae7ac, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/313987ad62dd4842a7011c9fb47c05b7, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/dd50874d686941e9972d021dc9a869d6, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/37a732cae24f460797368f4d16a5ea11, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/3f5e5640679f40d798e5463bf964eaba, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/17913e69c8054e69830264c1508cf353, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/e9d3253b129d425ea646be0fdfd9de58, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/203737f140274d7fa679510220914d05, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/8baa731b18d84c399d5f2a96027c0781, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/013d6487fbec401393b216c551c18d84, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/7337d9279f954897bc434aed573f8aec, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/d0910d4aecb843c3b86db8a0ebea9ca2, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/d48fc12ff131488fa5d791a6d18bf905, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/ee1c54e248dd409b823f78ab872a0056, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/3c33325793c643e1b24eba94ac33aec5, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/3a388032a98f4be587e08cba96858dc4, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/394c8fcd86f546fa9c6ee1440f209d0e, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/575a30ad120e4a5c96a4631bc6c69330, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/3ddbcb1a226d449197f0ae5962faf468, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/8d08e26db09f4251878128b2bb8b42bf, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/3c3a0a9c77344505b66abe8a76b31219, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/b762c72a64514bdeb3aa101a5853d6d2, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/3aad82271d474a698f809a27acd4afd6] to archive 2024-11-23T14:27:05,275 DEBUG [StoreCloser-TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-23T14:27:05,276 DEBUG [StoreCloser-TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/fc69506ab2b048cf9019543873e0a017 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/fc69506ab2b048cf9019543873e0a017 2024-11-23T14:27:05,277 DEBUG [StoreCloser-TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/b77d2da995604fd59e03f54dd9ed9297 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/b77d2da995604fd59e03f54dd9ed9297 2024-11-23T14:27:05,277 DEBUG [StoreCloser-TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/a897475d1eea46c29c3ca2cd1c3bc7be to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/a897475d1eea46c29c3ca2cd1c3bc7be 2024-11-23T14:27:05,278 DEBUG [StoreCloser-TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/cc893cfd2fb548b3a42a1047b872a8c3 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/cc893cfd2fb548b3a42a1047b872a8c3 2024-11-23T14:27:05,279 DEBUG [StoreCloser-TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/f4500ed0cbc94b57aefafb5e08e6f5a2 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/f4500ed0cbc94b57aefafb5e08e6f5a2 2024-11-23T14:27:05,280 DEBUG [StoreCloser-TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/419ea993d355482ab74361efef7ae7ac to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/419ea993d355482ab74361efef7ae7ac 2024-11-23T14:27:05,281 DEBUG [StoreCloser-TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/313987ad62dd4842a7011c9fb47c05b7 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/313987ad62dd4842a7011c9fb47c05b7 2024-11-23T14:27:05,282 DEBUG [StoreCloser-TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/dd50874d686941e9972d021dc9a869d6 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/dd50874d686941e9972d021dc9a869d6 2024-11-23T14:27:05,283 DEBUG [StoreCloser-TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/37a732cae24f460797368f4d16a5ea11 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/37a732cae24f460797368f4d16a5ea11 2024-11-23T14:27:05,286 DEBUG [StoreCloser-TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/3f5e5640679f40d798e5463bf964eaba to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/3f5e5640679f40d798e5463bf964eaba 2024-11-23T14:27:05,288 DEBUG [StoreCloser-TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/17913e69c8054e69830264c1508cf353 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/17913e69c8054e69830264c1508cf353 2024-11-23T14:27:05,289 DEBUG [StoreCloser-TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/e9d3253b129d425ea646be0fdfd9de58 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/e9d3253b129d425ea646be0fdfd9de58 2024-11-23T14:27:05,290 DEBUG [StoreCloser-TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/203737f140274d7fa679510220914d05 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/203737f140274d7fa679510220914d05 2024-11-23T14:27:05,291 DEBUG [StoreCloser-TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/8baa731b18d84c399d5f2a96027c0781 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/8baa731b18d84c399d5f2a96027c0781 2024-11-23T14:27:05,292 DEBUG [StoreCloser-TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/013d6487fbec401393b216c551c18d84 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/013d6487fbec401393b216c551c18d84 2024-11-23T14:27:05,293 DEBUG [StoreCloser-TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/7337d9279f954897bc434aed573f8aec to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/7337d9279f954897bc434aed573f8aec 2024-11-23T14:27:05,295 DEBUG [StoreCloser-TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/d0910d4aecb843c3b86db8a0ebea9ca2 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/d0910d4aecb843c3b86db8a0ebea9ca2 2024-11-23T14:27:05,296 DEBUG [StoreCloser-TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/d48fc12ff131488fa5d791a6d18bf905 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/d48fc12ff131488fa5d791a6d18bf905 2024-11-23T14:27:05,299 DEBUG [StoreCloser-TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/ee1c54e248dd409b823f78ab872a0056 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/ee1c54e248dd409b823f78ab872a0056 2024-11-23T14:27:05,300 DEBUG [StoreCloser-TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/3c33325793c643e1b24eba94ac33aec5 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/3c33325793c643e1b24eba94ac33aec5 2024-11-23T14:27:05,301 DEBUG [StoreCloser-TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/3a388032a98f4be587e08cba96858dc4 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/3a388032a98f4be587e08cba96858dc4 2024-11-23T14:27:05,303 DEBUG [StoreCloser-TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/394c8fcd86f546fa9c6ee1440f209d0e to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/394c8fcd86f546fa9c6ee1440f209d0e 2024-11-23T14:27:05,304 DEBUG [StoreCloser-TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/575a30ad120e4a5c96a4631bc6c69330 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/575a30ad120e4a5c96a4631bc6c69330 2024-11-23T14:27:05,305 DEBUG [StoreCloser-TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/3ddbcb1a226d449197f0ae5962faf468 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/3ddbcb1a226d449197f0ae5962faf468 2024-11-23T14:27:05,306 DEBUG [StoreCloser-TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/8d08e26db09f4251878128b2bb8b42bf to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/8d08e26db09f4251878128b2bb8b42bf 2024-11-23T14:27:05,307 DEBUG [StoreCloser-TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/3c3a0a9c77344505b66abe8a76b31219 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/3c3a0a9c77344505b66abe8a76b31219 2024-11-23T14:27:05,308 DEBUG [StoreCloser-TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/b762c72a64514bdeb3aa101a5853d6d2 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/b762c72a64514bdeb3aa101a5853d6d2 2024-11-23T14:27:05,309 DEBUG [StoreCloser-TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/3aad82271d474a698f809a27acd4afd6 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/3aad82271d474a698f809a27acd4afd6 2024-11-23T14:27:05,310 DEBUG [StoreCloser-TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/5cdba96e3153415ea6a1f22ef7dac93c, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/977d1672caee41578b7c48dedea8169b, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/61f36d55bb0949948b9f763819380aab, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/f8e58cc8ce944f50aee652d8df260ec1, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/4e2c618d910b4d4fa7e1da0974b5708f, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/c3a02fdf495c45c19d9d5ba2b579e842, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/18a624705a254bf99c7c8e7bbea86069, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/73d8814609584989a07281c06edc12f1, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/41abb2539dbb41e1a7253f096311648b, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/7d20fb2343a14c5c89e32337948bc156, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/33704be76b6e4bd996078a7193069229, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/c77a33b7c2a04ff9b318b0dab8c424aa, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/44cd185037fa4277a4a53d56e1672f3d, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/e1322d26948c47d288eca4b8e1bd4edc, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/d4e974bb019e4ef8a87d3b758859d8a9, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/65bb71c263964fdeb0fa0cccdf8b936c, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/c74d35807c714464947596ce87d0824a, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/2755ecb8e6f1495e9383084a9665e257, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/174b6d59aec34f549fae7965ad77926e, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/859aa3cc415444638298019da215234a, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/1be829ff64c74c919b1781ea5e873d07, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/ebbfb4822a634c2f866340149d489832, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/5fec32ffb0ba41ef8b17f29acc847807, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/c21fdccb6eca49ac9f47b4583e089c8a, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/a7e21b5ba2834ddca414c9b17bf1a105, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/5b05f4829a23462199da18a78d0c444c, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/4d3b9508600c481fb25d4f0e1b833040] to archive 2024-11-23T14:27:05,312 DEBUG [StoreCloser-TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-23T14:27:05,313 DEBUG [StoreCloser-TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/5cdba96e3153415ea6a1f22ef7dac93c to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/5cdba96e3153415ea6a1f22ef7dac93c 2024-11-23T14:27:05,314 DEBUG [StoreCloser-TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/977d1672caee41578b7c48dedea8169b to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/977d1672caee41578b7c48dedea8169b 2024-11-23T14:27:05,315 DEBUG [StoreCloser-TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/61f36d55bb0949948b9f763819380aab to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/61f36d55bb0949948b9f763819380aab 2024-11-23T14:27:05,316 DEBUG [StoreCloser-TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/f8e58cc8ce944f50aee652d8df260ec1 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/f8e58cc8ce944f50aee652d8df260ec1 2024-11-23T14:27:05,317 DEBUG [StoreCloser-TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/4e2c618d910b4d4fa7e1da0974b5708f to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/4e2c618d910b4d4fa7e1da0974b5708f 2024-11-23T14:27:05,318 DEBUG [StoreCloser-TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/c3a02fdf495c45c19d9d5ba2b579e842 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/c3a02fdf495c45c19d9d5ba2b579e842 2024-11-23T14:27:05,319 DEBUG [StoreCloser-TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/18a624705a254bf99c7c8e7bbea86069 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/18a624705a254bf99c7c8e7bbea86069 2024-11-23T14:27:05,320 DEBUG [StoreCloser-TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/73d8814609584989a07281c06edc12f1 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/73d8814609584989a07281c06edc12f1 2024-11-23T14:27:05,321 DEBUG [StoreCloser-TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/41abb2539dbb41e1a7253f096311648b to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/41abb2539dbb41e1a7253f096311648b 2024-11-23T14:27:05,322 DEBUG [StoreCloser-TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/7d20fb2343a14c5c89e32337948bc156 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/7d20fb2343a14c5c89e32337948bc156 2024-11-23T14:27:05,323 DEBUG [StoreCloser-TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/33704be76b6e4bd996078a7193069229 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/33704be76b6e4bd996078a7193069229 2024-11-23T14:27:05,323 DEBUG [StoreCloser-TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/c77a33b7c2a04ff9b318b0dab8c424aa to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/c77a33b7c2a04ff9b318b0dab8c424aa 2024-11-23T14:27:05,324 DEBUG [StoreCloser-TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/44cd185037fa4277a4a53d56e1672f3d to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/44cd185037fa4277a4a53d56e1672f3d 2024-11-23T14:27:05,325 DEBUG [StoreCloser-TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/e1322d26948c47d288eca4b8e1bd4edc to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/e1322d26948c47d288eca4b8e1bd4edc 2024-11-23T14:27:05,326 DEBUG [StoreCloser-TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/d4e974bb019e4ef8a87d3b758859d8a9 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/d4e974bb019e4ef8a87d3b758859d8a9 2024-11-23T14:27:05,327 DEBUG [StoreCloser-TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/65bb71c263964fdeb0fa0cccdf8b936c to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/65bb71c263964fdeb0fa0cccdf8b936c 2024-11-23T14:27:05,328 DEBUG [StoreCloser-TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/c74d35807c714464947596ce87d0824a to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/c74d35807c714464947596ce87d0824a 2024-11-23T14:27:05,329 DEBUG [StoreCloser-TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/2755ecb8e6f1495e9383084a9665e257 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/2755ecb8e6f1495e9383084a9665e257 2024-11-23T14:27:05,330 DEBUG [StoreCloser-TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/174b6d59aec34f549fae7965ad77926e to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/174b6d59aec34f549fae7965ad77926e 2024-11-23T14:27:05,330 DEBUG [StoreCloser-TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/859aa3cc415444638298019da215234a to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/859aa3cc415444638298019da215234a 2024-11-23T14:27:05,331 DEBUG [StoreCloser-TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/1be829ff64c74c919b1781ea5e873d07 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/1be829ff64c74c919b1781ea5e873d07 2024-11-23T14:27:05,332 DEBUG [StoreCloser-TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/ebbfb4822a634c2f866340149d489832 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/ebbfb4822a634c2f866340149d489832 2024-11-23T14:27:05,333 DEBUG [StoreCloser-TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/5fec32ffb0ba41ef8b17f29acc847807 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/5fec32ffb0ba41ef8b17f29acc847807 2024-11-23T14:27:05,334 DEBUG [StoreCloser-TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/c21fdccb6eca49ac9f47b4583e089c8a to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/c21fdccb6eca49ac9f47b4583e089c8a 2024-11-23T14:27:05,335 DEBUG [StoreCloser-TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/a7e21b5ba2834ddca414c9b17bf1a105 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/a7e21b5ba2834ddca414c9b17bf1a105 2024-11-23T14:27:05,335 DEBUG [StoreCloser-TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/5b05f4829a23462199da18a78d0c444c to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/5b05f4829a23462199da18a78d0c444c 2024-11-23T14:27:05,336 DEBUG [StoreCloser-TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/4d3b9508600c481fb25d4f0e1b833040 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/4d3b9508600c481fb25d4f0e1b833040 2024-11-23T14:27:05,340 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/recovered.edits/418.seqid, newMaxSeqId=418, maxSeqId=4 2024-11-23T14:27:05,341 INFO [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. 2024-11-23T14:27:05,341 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1676): Region close journal for fa04d9e7f6ce6ae3964ded9c7627848b: Waiting for close lock at 1732372023975Running coprocessor pre-close hooks at 1732372023975Disabling compacts and flushes for region at 1732372023975Disabling writes for close at 1732372023975Obtaining lock to block concurrent updates at 1732372023975Preparing flush snapshotting stores in fa04d9e7f6ce6ae3964ded9c7627848b at 1732372023975Finished memstore snapshotting TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1732372023976 (+1 ms)Flushing stores of TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b. at 1732372023980 (+4 ms)Flushing fa04d9e7f6ce6ae3964ded9c7627848b/A: creating writer at 1732372023980Flushing mob file fa04d9e7f6ce6ae3964ded9c7627848b/A: appending metadata at 1732372023991 (+11 ms)Flushing mob file fa04d9e7f6ce6ae3964ded9c7627848b/A: closing flushed file at 1732372023991Flushing fa04d9e7f6ce6ae3964ded9c7627848b/A: appending metadata at 1732372024410 (+419 ms)Flushing fa04d9e7f6ce6ae3964ded9c7627848b/A: closing flushed file at 1732372024410Flushing fa04d9e7f6ce6ae3964ded9c7627848b/B: creating writer at 1732372024418 (+8 ms)Flushing fa04d9e7f6ce6ae3964ded9c7627848b/B: appending metadata at 1732372024420 (+2 ms)Flushing fa04d9e7f6ce6ae3964ded9c7627848b/B: closing flushed file at 1732372024420Flushing fa04d9e7f6ce6ae3964ded9c7627848b/C: creating writer at 1732372024827 (+407 ms)Flushing fa04d9e7f6ce6ae3964ded9c7627848b/C: appending metadata at 1732372024830 (+3 ms)Flushing fa04d9e7f6ce6ae3964ded9c7627848b/C: closing flushed file at 1732372024830Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@726750c5: reopening flushed file at 1732372025238 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4410c33c: reopening flushed file at 1732372025242 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3f636a2: reopening flushed file at 1732372025245 (+3 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for fa04d9e7f6ce6ae3964ded9c7627848b in 1275ms, sequenceid=415, compaction requested=true at 1732372025250 (+5 ms)Writing region close event to WAL at 1732372025337 (+87 ms)Running coprocessor post-close hooks at 1732372025341 (+4 ms)Closed at 1732372025341 2024-11-23T14:27:05,343 INFO [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] handler.UnassignRegionHandler(157): Closed fa04d9e7f6ce6ae3964ded9c7627848b 2024-11-23T14:27:05,344 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=121 updating hbase:meta row=fa04d9e7f6ce6ae3964ded9c7627848b, regionState=CLOSED 2024-11-23T14:27:05,345 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=122, ppid=121, state=RUNNABLE, hasLock=false; CloseRegionProcedure fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 because future has completed 2024-11-23T14:27:05,348 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-23T14:27:05,348 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; CloseRegionProcedure fa04d9e7f6ce6ae3964ded9c7627848b, server=c6f7b1897123,35385,1732371902767 in 1.5250 sec 2024-11-23T14:27:05,350 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=121, resume processing ppid=120 2024-11-23T14:27:05,350 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, ppid=120, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fa04d9e7f6ce6ae3964ded9c7627848b, UNASSIGN in 1.5360 sec 2024-11-23T14:27:05,351 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-23T14:27:05,351 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5400 sec 2024-11-23T14:27:05,353 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732372025352"}]},"ts":"1732372025352"} 2024-11-23T14:27:05,354 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-23T14:27:05,354 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-23T14:27:05,356 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5520 sec 2024-11-23T14:27:05,804 WARN [regionserver/c6f7b1897123:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 1, running: 1 2024-11-23T14:27:05,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-23T14:27:05,950 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-23T14:27:05,951 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-23T14:27:05,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T14:27:05,952 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=123, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T14:27:05,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-23T14:27:05,953 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=123, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T14:27:05,955 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b 2024-11-23T14:27:05,956 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A, FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B, FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C, FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/recovered.edits] 2024-11-23T14:27:05,958 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/520dbaeddf0c417c8124c1a615edfdc2 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/520dbaeddf0c417c8124c1a615edfdc2 2024-11-23T14:27:05,959 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/5a13cdb2806e4b9a9456d81ba42a4838 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/5a13cdb2806e4b9a9456d81ba42a4838 2024-11-23T14:27:05,960 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/6ab6a8a00d70472e929bff2193866cc6 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/A/6ab6a8a00d70472e929bff2193866cc6 2024-11-23T14:27:05,963 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/a6b6118637104b81a263f0b23eb44f61 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/a6b6118637104b81a263f0b23eb44f61 2024-11-23T14:27:05,964 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/c8b627902d0a4ae2bdac5717a132430f to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/c8b627902d0a4ae2bdac5717a132430f 2024-11-23T14:27:05,965 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/f61514ed1bef4550b0cbbb76ea91d473 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/B/f61514ed1bef4550b0cbbb76ea91d473 2024-11-23T14:27:05,967 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/0908cde42e4c42f3810ed1f30dd1ac2c to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/0908cde42e4c42f3810ed1f30dd1ac2c 2024-11-23T14:27:05,968 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/162b02b59a5547bc882a93a75bf93f0e to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/162b02b59a5547bc882a93a75bf93f0e 2024-11-23T14:27:05,969 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/347bcc25b773463ca88e2e4c22af2912 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/C/347bcc25b773463ca88e2e4c22af2912 2024-11-23T14:27:05,971 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/recovered.edits/418.seqid to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b/recovered.edits/418.seqid 2024-11-23T14:27:05,971 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/fa04d9e7f6ce6ae3964ded9c7627848b 2024-11-23T14:27:05,971 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-23T14:27:05,972 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-23T14:27:05,972 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-23T14:27:05,974 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112300f1b1e179634fc386c36985405647ac_fa04d9e7f6ce6ae3964ded9c7627848b to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112300f1b1e179634fc386c36985405647ac_fa04d9e7f6ce6ae3964ded9c7627848b 2024-11-23T14:27:05,975 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411230a0ac9eaabbf4c11a9c19aa99c73c2ce_fa04d9e7f6ce6ae3964ded9c7627848b to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411230a0ac9eaabbf4c11a9c19aa99c73c2ce_fa04d9e7f6ce6ae3964ded9c7627848b 2024-11-23T14:27:05,976 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411231a0d67a3b21b46bfa0200ede5a5c15bd_fa04d9e7f6ce6ae3964ded9c7627848b to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411231a0d67a3b21b46bfa0200ede5a5c15bd_fa04d9e7f6ce6ae3964ded9c7627848b 2024-11-23T14:27:05,977 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411231fc2cab5c016494e812e4d67b016a277_fa04d9e7f6ce6ae3964ded9c7627848b to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411231fc2cab5c016494e812e4d67b016a277_fa04d9e7f6ce6ae3964ded9c7627848b 2024-11-23T14:27:05,977 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123285a9d7db40c4dd59900fd55df525df0_fa04d9e7f6ce6ae3964ded9c7627848b to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123285a9d7db40c4dd59900fd55df525df0_fa04d9e7f6ce6ae3964ded9c7627848b 2024-11-23T14:27:05,978 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411232cc522e57250483f87e01c1e6a6c5e5a_fa04d9e7f6ce6ae3964ded9c7627848b to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411232cc522e57250483f87e01c1e6a6c5e5a_fa04d9e7f6ce6ae3964ded9c7627848b 2024-11-23T14:27:05,979 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411233c7e0a603335474098de9e3eb98ccfc9_fa04d9e7f6ce6ae3964ded9c7627848b to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411233c7e0a603335474098de9e3eb98ccfc9_fa04d9e7f6ce6ae3964ded9c7627848b 2024-11-23T14:27:05,980 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411234e6225dfc4744cd0b7a4c64fed02a4b1_fa04d9e7f6ce6ae3964ded9c7627848b to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411234e6225dfc4744cd0b7a4c64fed02a4b1_fa04d9e7f6ce6ae3964ded9c7627848b 2024-11-23T14:27:05,981 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112375ff32815055409a9657ef170ff4a07d_fa04d9e7f6ce6ae3964ded9c7627848b to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112375ff32815055409a9657ef170ff4a07d_fa04d9e7f6ce6ae3964ded9c7627848b 2024-11-23T14:27:05,982 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123787b54a1f8ee4a04a73f058d85bb9f73_fa04d9e7f6ce6ae3964ded9c7627848b to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123787b54a1f8ee4a04a73f058d85bb9f73_fa04d9e7f6ce6ae3964ded9c7627848b 2024-11-23T14:27:05,982 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411237f58e5f04d3545b9b1e5a943158cb2d5_fa04d9e7f6ce6ae3964ded9c7627848b to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411237f58e5f04d3545b9b1e5a943158cb2d5_fa04d9e7f6ce6ae3964ded9c7627848b 2024-11-23T14:27:05,983 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411239345fc7e3f0346c1b478517cddd89e5c_fa04d9e7f6ce6ae3964ded9c7627848b to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411239345fc7e3f0346c1b478517cddd89e5c_fa04d9e7f6ce6ae3964ded9c7627848b 2024-11-23T14:27:05,984 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123a6a24b1f38344df09c222fdb0a16acbf_fa04d9e7f6ce6ae3964ded9c7627848b to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123a6a24b1f38344df09c222fdb0a16acbf_fa04d9e7f6ce6ae3964ded9c7627848b 2024-11-23T14:27:05,985 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123b202aba90e64400183d3577ad6e8921b_fa04d9e7f6ce6ae3964ded9c7627848b to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123b202aba90e64400183d3577ad6e8921b_fa04d9e7f6ce6ae3964ded9c7627848b 2024-11-23T14:27:05,986 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123cd7e22eb6617466db192730c54e735df_fa04d9e7f6ce6ae3964ded9c7627848b to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123cd7e22eb6617466db192730c54e735df_fa04d9e7f6ce6ae3964ded9c7627848b 2024-11-23T14:27:05,987 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123cf36417990e54ab5a7da49c1002c7511_fa04d9e7f6ce6ae3964ded9c7627848b to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123cf36417990e54ab5a7da49c1002c7511_fa04d9e7f6ce6ae3964ded9c7627848b 2024-11-23T14:27:05,988 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123da270fc84e77450b8a7fdc825c3692cb_fa04d9e7f6ce6ae3964ded9c7627848b to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123da270fc84e77450b8a7fdc825c3692cb_fa04d9e7f6ce6ae3964ded9c7627848b 2024-11-23T14:27:05,989 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123df5c33f0f28d470098935323957df4c9_fa04d9e7f6ce6ae3964ded9c7627848b to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123df5c33f0f28d470098935323957df4c9_fa04d9e7f6ce6ae3964ded9c7627848b 2024-11-23T14:27:05,990 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123e1161c19b6cf4d74abb8af1da63f0109_fa04d9e7f6ce6ae3964ded9c7627848b to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123e1161c19b6cf4d74abb8af1da63f0109_fa04d9e7f6ce6ae3964ded9c7627848b 2024-11-23T14:27:05,991 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123e1d86241c93f4360ac54feaa88e680a0_fa04d9e7f6ce6ae3964ded9c7627848b to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123e1d86241c93f4360ac54feaa88e680a0_fa04d9e7f6ce6ae3964ded9c7627848b 2024-11-23T14:27:05,992 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123eaab989d68df4fd089421aa8204830bb_fa04d9e7f6ce6ae3964ded9c7627848b to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123eaab989d68df4fd089421aa8204830bb_fa04d9e7f6ce6ae3964ded9c7627848b 2024-11-23T14:27:05,993 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123ee89fb4721e14893acb020bd8709195f_fa04d9e7f6ce6ae3964ded9c7627848b to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123ee89fb4721e14893acb020bd8709195f_fa04d9e7f6ce6ae3964ded9c7627848b 2024-11-23T14:27:05,993 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-23T14:27:05,995 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=123, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T14:27:05,997 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-23T14:27:06,000 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-23T14:27:06,001 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=123, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T14:27:06,001 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-23T14:27:06,001 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732372026001"}]},"ts":"9223372036854775807"} 2024-11-23T14:27:06,003 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-23T14:27:06,003 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => fa04d9e7f6ce6ae3964ded9c7627848b, NAME => 'TestAcidGuarantees,,1732371997655.fa04d9e7f6ce6ae3964ded9c7627848b.', STARTKEY => '', ENDKEY => ''}] 2024-11-23T14:27:06,003 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-23T14:27:06,004 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732372026003"}]},"ts":"9223372036854775807"} 2024-11-23T14:27:06,005 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-23T14:27:06,006 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=123, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T14:27:06,007 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 55 msec 2024-11-23T14:27:06,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-23T14:27:06,060 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-23T14:27:06,060 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-23T14:27:06,070 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=240 (was 240), OpenFileDescriptor=453 (was 451) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=630 (was 518) - SystemLoadAverage LEAK? -, ProcessCount=12 (was 11) - ProcessCount LEAK? -, AvailableMemoryMB=3180 (was 3405) 2024-11-23T14:27:06,078 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=240, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=630, ProcessCount=12, AvailableMemoryMB=3179 2024-11-23T14:27:06,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-23T14:27:06,080 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-23T14:27:06,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] procedure2.ProcedureExecutor(1139): Stored pid=124, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-23T14:27:06,082 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-23T14:27:06,082 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:06,082 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 124 2024-11-23T14:27:06,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-11-23T14:27:06,083 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-23T14:27:06,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742310_1486 (size=959) 2024-11-23T14:27:06,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-11-23T14:27:06,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-11-23T14:27:06,492 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a 2024-11-23T14:27:06,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742311_1487 (size=53) 2024-11-23T14:27:06,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-11-23T14:27:06,901 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T14:27:06,901 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 3d5e64ff6f2c8eec1637e7e45ace6102, disabling compactions & flushes 2024-11-23T14:27:06,901 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:06,901 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:06,901 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. after waiting 0 ms 2024-11-23T14:27:06,901 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:06,901 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:06,901 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 3d5e64ff6f2c8eec1637e7e45ace6102: Waiting for close lock at 1732372026901Disabling compacts and flushes for region at 1732372026901Disabling writes for close at 1732372026901Writing region close event to WAL at 1732372026901Closed at 1732372026901 2024-11-23T14:27:06,902 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-23T14:27:06,902 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732372026902"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732372026902"}]},"ts":"1732372026902"} 2024-11-23T14:27:06,903 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-23T14:27:06,904 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-23T14:27:06,904 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732372026904"}]},"ts":"1732372026904"} 2024-11-23T14:27:06,906 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-23T14:27:06,906 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3d5e64ff6f2c8eec1637e7e45ace6102, ASSIGN}] 2024-11-23T14:27:06,907 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3d5e64ff6f2c8eec1637e7e45ace6102, ASSIGN 2024-11-23T14:27:06,907 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3d5e64ff6f2c8eec1637e7e45ace6102, ASSIGN; state=OFFLINE, location=c6f7b1897123,35385,1732371902767; forceNewPlan=false, retain=false 2024-11-23T14:27:07,058 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=125 updating hbase:meta row=3d5e64ff6f2c8eec1637e7e45ace6102, regionState=OPENING, regionLocation=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:07,061 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3d5e64ff6f2c8eec1637e7e45ace6102, ASSIGN because future has completed 2024-11-23T14:27:07,062 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767}] 2024-11-23T14:27:07,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-11-23T14:27:07,223 INFO [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:07,223 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(7752): Opening region: {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} 2024-11-23T14:27:07,224 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 3d5e64ff6f2c8eec1637e7e45ace6102 2024-11-23T14:27:07,225 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T14:27:07,225 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(7794): checking encryption for 3d5e64ff6f2c8eec1637e7e45ace6102 2024-11-23T14:27:07,225 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(7797): checking classloading for 3d5e64ff6f2c8eec1637e7e45ace6102 2024-11-23T14:27:07,227 INFO [StoreOpener-3d5e64ff6f2c8eec1637e7e45ace6102-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 3d5e64ff6f2c8eec1637e7e45ace6102 2024-11-23T14:27:07,229 INFO [StoreOpener-3d5e64ff6f2c8eec1637e7e45ace6102-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3d5e64ff6f2c8eec1637e7e45ace6102 columnFamilyName A 2024-11-23T14:27:07,229 DEBUG [StoreOpener-3d5e64ff6f2c8eec1637e7e45ace6102-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:07,229 INFO [StoreOpener-3d5e64ff6f2c8eec1637e7e45ace6102-1 {}] regionserver.HStore(327): Store=3d5e64ff6f2c8eec1637e7e45ace6102/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T14:27:07,230 INFO [StoreOpener-3d5e64ff6f2c8eec1637e7e45ace6102-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 3d5e64ff6f2c8eec1637e7e45ace6102 2024-11-23T14:27:07,231 INFO [StoreOpener-3d5e64ff6f2c8eec1637e7e45ace6102-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3d5e64ff6f2c8eec1637e7e45ace6102 columnFamilyName B 2024-11-23T14:27:07,231 DEBUG [StoreOpener-3d5e64ff6f2c8eec1637e7e45ace6102-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:07,232 INFO [StoreOpener-3d5e64ff6f2c8eec1637e7e45ace6102-1 {}] regionserver.HStore(327): Store=3d5e64ff6f2c8eec1637e7e45ace6102/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T14:27:07,232 INFO [StoreOpener-3d5e64ff6f2c8eec1637e7e45ace6102-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 3d5e64ff6f2c8eec1637e7e45ace6102 2024-11-23T14:27:07,234 INFO [StoreOpener-3d5e64ff6f2c8eec1637e7e45ace6102-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3d5e64ff6f2c8eec1637e7e45ace6102 columnFamilyName C 2024-11-23T14:27:07,234 DEBUG [StoreOpener-3d5e64ff6f2c8eec1637e7e45ace6102-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:07,234 INFO [StoreOpener-3d5e64ff6f2c8eec1637e7e45ace6102-1 {}] regionserver.HStore(327): Store=3d5e64ff6f2c8eec1637e7e45ace6102/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T14:27:07,235 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1038): replaying wal for 3d5e64ff6f2c8eec1637e7e45ace6102 2024-11-23T14:27:07,236 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102 2024-11-23T14:27:07,236 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102 2024-11-23T14:27:07,238 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1048): stopping wal replay for 3d5e64ff6f2c8eec1637e7e45ace6102 2024-11-23T14:27:07,238 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1060): Cleaning up temporary data for 3d5e64ff6f2c8eec1637e7e45ace6102 2024-11-23T14:27:07,239 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-23T14:27:07,240 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1093): writing seq id for 3d5e64ff6f2c8eec1637e7e45ace6102 2024-11-23T14:27:07,243 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T14:27:07,244 INFO [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1114): Opened 3d5e64ff6f2c8eec1637e7e45ace6102; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60365677, jitterRate=-0.100481316447258}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-23T14:27:07,244 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3d5e64ff6f2c8eec1637e7e45ace6102 2024-11-23T14:27:07,245 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1006): Region open journal for 3d5e64ff6f2c8eec1637e7e45ace6102: Running coprocessor pre-open hook at 1732372027225Writing region info on filesystem at 1732372027225Initializing all the Stores at 1732372027226 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732372027226Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732372027227 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732372027227Cleaning up temporary data from old regions at 1732372027238 (+11 ms)Running coprocessor post-open hooks at 1732372027244 (+6 ms)Region opened successfully at 1732372027245 (+1 ms) 2024-11-23T14:27:07,246 INFO [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., pid=126, masterSystemTime=1732372027216 2024-11-23T14:27:07,248 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:07,248 INFO [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:07,249 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=125 updating hbase:meta row=3d5e64ff6f2c8eec1637e7e45ace6102, regionState=OPEN, openSeqNum=2, regionLocation=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:07,251 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=126, ppid=125, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 because future has completed 2024-11-23T14:27:07,255 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-23T14:27:07,255 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; OpenRegionProcedure 3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 in 191 msec 2024-11-23T14:27:07,258 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=125, resume processing ppid=124 2024-11-23T14:27:07,258 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, ppid=124, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3d5e64ff6f2c8eec1637e7e45ace6102, ASSIGN in 349 msec 2024-11-23T14:27:07,259 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-23T14:27:07,259 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732372027259"}]},"ts":"1732372027259"} 2024-11-23T14:27:07,261 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-23T14:27:07,262 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-23T14:27:07,265 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1830 sec 2024-11-23T14:27:08,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-11-23T14:27:08,231 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-23T14:27:08,233 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@460158be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:27:08,233 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6f7b1897123,38947,-1 for getting cluster id 2024-11-23T14:27:08,233 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T14:27:08,234 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c88da961-458f-4375-b9d3-275f7f90019e' 2024-11-23T14:27:08,234 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T14:27:08,234 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c88da961-458f-4375-b9d3-275f7f90019e" 2024-11-23T14:27:08,234 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f13967c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:27:08,234 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6f7b1897123,38947,-1] 2024-11-23T14:27:08,235 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T14:27:08,235 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:27:08,236 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42172, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T14:27:08,236 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f5e046a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:27:08,237 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T14:27:08,238 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6f7b1897123,35385,1732371902767, seqNum=-1] 2024-11-23T14:27:08,238 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T14:27:08,239 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32774, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T14:27:08,240 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-23T14:27:08,241 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is c6f7b1897123,38947,1732371901705 2024-11-23T14:27:08,241 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@398ab390 2024-11-23T14:27:08,241 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-23T14:27:08,242 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42174, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-23T14:27:08,243 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5159e3f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:27:08,243 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6f7b1897123,38947,-1 for getting cluster id 2024-11-23T14:27:08,243 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T14:27:08,244 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c88da961-458f-4375-b9d3-275f7f90019e' 2024-11-23T14:27:08,244 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T14:27:08,244 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c88da961-458f-4375-b9d3-275f7f90019e" 2024-11-23T14:27:08,244 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@782e7570, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:27:08,244 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6f7b1897123,38947,-1] 2024-11-23T14:27:08,244 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T14:27:08,244 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:27:08,245 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42186, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T14:27:08,245 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43fc71cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:27:08,246 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1062bfdd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:27:08,246 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6f7b1897123,38947,-1 for getting cluster id 2024-11-23T14:27:08,246 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T14:27:08,246 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c88da961-458f-4375-b9d3-275f7f90019e' 2024-11-23T14:27:08,247 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T14:27:08,247 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c88da961-458f-4375-b9d3-275f7f90019e" 2024-11-23T14:27:08,247 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ddae05f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:27:08,247 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6f7b1897123,38947,-1] 2024-11-23T14:27:08,247 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T14:27:08,247 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:27:08,248 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42198, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T14:27:08,248 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f6d591a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:27:08,249 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d7232e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:27:08,249 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6f7b1897123,38947,-1 for getting cluster id 2024-11-23T14:27:08,249 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T14:27:08,249 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c88da961-458f-4375-b9d3-275f7f90019e' 2024-11-23T14:27:08,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T14:27:08,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c88da961-458f-4375-b9d3-275f7f90019e" 2024-11-23T14:27:08,250 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a04078e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:27:08,250 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6f7b1897123,38947,-1] 2024-11-23T14:27:08,250 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T14:27:08,250 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:27:08,251 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42216, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T14:27:08,251 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@562388bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:27:08,252 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39880b68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:27:08,252 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6f7b1897123,38947,-1 for getting cluster id 2024-11-23T14:27:08,252 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T14:27:08,253 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c88da961-458f-4375-b9d3-275f7f90019e' 2024-11-23T14:27:08,253 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T14:27:08,253 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c88da961-458f-4375-b9d3-275f7f90019e" 2024-11-23T14:27:08,253 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57391fd2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:27:08,253 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6f7b1897123,38947,-1] 2024-11-23T14:27:08,253 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T14:27:08,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:27:08,254 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42238, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T14:27:08,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e83ba1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:27:08,256 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67f0c5cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:27:08,256 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6f7b1897123,38947,-1 for getting cluster id 2024-11-23T14:27:08,256 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T14:27:08,256 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c88da961-458f-4375-b9d3-275f7f90019e' 2024-11-23T14:27:08,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T14:27:08,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c88da961-458f-4375-b9d3-275f7f90019e" 2024-11-23T14:27:08,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45b3483, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:27:08,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6f7b1897123,38947,-1] 2024-11-23T14:27:08,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T14:27:08,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:27:08,257 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42266, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T14:27:08,258 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36694032, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:27:08,258 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1593fd05, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:27:08,258 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6f7b1897123,38947,-1 for getting cluster id 2024-11-23T14:27:08,259 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T14:27:08,259 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c88da961-458f-4375-b9d3-275f7f90019e' 2024-11-23T14:27:08,259 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T14:27:08,259 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c88da961-458f-4375-b9d3-275f7f90019e" 2024-11-23T14:27:08,259 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78b398fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:27:08,259 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6f7b1897123,38947,-1] 2024-11-23T14:27:08,260 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T14:27:08,260 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:27:08,260 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42282, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T14:27:08,261 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@284e7532, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:27:08,261 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@729114ff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:27:08,261 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6f7b1897123,38947,-1 for getting cluster id 2024-11-23T14:27:08,261 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T14:27:08,262 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c88da961-458f-4375-b9d3-275f7f90019e' 2024-11-23T14:27:08,262 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T14:27:08,262 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c88da961-458f-4375-b9d3-275f7f90019e" 2024-11-23T14:27:08,262 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c76aed3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:27:08,262 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6f7b1897123,38947,-1] 2024-11-23T14:27:08,262 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T14:27:08,263 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:27:08,263 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42294, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T14:27:08,263 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38df3817, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:27:08,264 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@139f4c00, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:27:08,264 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6f7b1897123,38947,-1 for getting cluster id 2024-11-23T14:27:08,264 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T14:27:08,265 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c88da961-458f-4375-b9d3-275f7f90019e' 2024-11-23T14:27:08,265 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T14:27:08,265 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c88da961-458f-4375-b9d3-275f7f90019e" 2024-11-23T14:27:08,265 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45120b19, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:27:08,265 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6f7b1897123,38947,-1] 2024-11-23T14:27:08,265 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T14:27:08,265 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:27:08,266 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42314, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T14:27:08,266 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8f5af45, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:27:08,267 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14b03bcc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:27:08,267 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6f7b1897123,38947,-1 for getting cluster id 2024-11-23T14:27:08,267 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T14:27:08,267 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c88da961-458f-4375-b9d3-275f7f90019e' 2024-11-23T14:27:08,267 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T14:27:08,267 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c88da961-458f-4375-b9d3-275f7f90019e" 2024-11-23T14:27:08,268 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f1c8547, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:27:08,268 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6f7b1897123,38947,-1] 2024-11-23T14:27:08,268 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T14:27:08,268 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:27:08,268 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42322, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T14:27:08,269 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23b97d13, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:27:08,269 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b0bbd77, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:27:08,269 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6f7b1897123,38947,-1 for getting cluster id 2024-11-23T14:27:08,269 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T14:27:08,270 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c88da961-458f-4375-b9d3-275f7f90019e' 2024-11-23T14:27:08,270 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T14:27:08,270 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c88da961-458f-4375-b9d3-275f7f90019e" 2024-11-23T14:27:08,270 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@398454cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:27:08,270 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6f7b1897123,38947,-1] 2024-11-23T14:27:08,270 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T14:27:08,270 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:27:08,271 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42336, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T14:27:08,271 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4628fe42, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:27:08,272 DEBUG [Thread-2183 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T14:27:08,272 DEBUG [Thread-2185 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T14:27:08,272 DEBUG [Thread-2174 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T14:27:08,272 DEBUG [Thread-2168 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T14:27:08,272 DEBUG [Thread-2170 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T14:27:08,273 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-23T14:27:08,273 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6f7b1897123,35385,1732371902767, seqNum=-1] 2024-11-23T14:27:08,273 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6f7b1897123,35385,1732371902767, seqNum=-1] 2024-11-23T14:27:08,273 DEBUG [Thread-2177 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T14:27:08,273 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6f7b1897123,35385,1732371902767, seqNum=-1] 2024-11-23T14:27:08,273 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T14:27:08,273 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T14:27:08,273 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T14:27:08,273 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6f7b1897123,35385,1732371902767, seqNum=-1] 2024-11-23T14:27:08,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-23T14:27:08,274 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T14:27:08,274 DEBUG [Thread-2177 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6f7b1897123,35385,1732371902767, seqNum=-1] 2024-11-23T14:27:08,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6f7b1897123,35385,1732371902767, seqNum=-1] 2024-11-23T14:27:08,274 DEBUG [Thread-2177 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T14:27:08,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T14:27:08,274 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T14:27:08,274 DEBUG [Thread-2181 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T14:27:08,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-23T14:27:08,275 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32790, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T14:27:08,275 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32798, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T14:27:08,275 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32788, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T14:27:08,275 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32810, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T14:27:08,275 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6f7b1897123,35385,1732371902767, seqNum=-1] 2024-11-23T14:27:08,275 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32824, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T14:27:08,275 DEBUG [Thread-2166 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T14:27:08,276 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T14:27:08,276 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32840, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T14:27:08,276 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T14:27:08,276 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T14:27:08,276 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6f7b1897123,35385,1732371902767, seqNum=-1] 2024-11-23T14:27:08,276 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T14:27:08,276 DEBUG [Thread-2172 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T14:27:08,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2] 2024-11-23T14:27:08,277 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32844, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T14:27:08,277 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32854, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T14:27:08,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2] 2024-11-23T14:27:08,278 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6f7b1897123,35385,1732371902767, seqNum=-1] 2024-11-23T14:27:08,278 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T14:27:08,278 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2] 2024-11-23T14:27:08,279 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32862, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T14:27:08,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2] 2024-11-23T14:27:08,280 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2] 2024-11-23T14:27:08,280 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2] 2024-11-23T14:27:08,281 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2] 2024-11-23T14:27:08,282 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2] 2024-11-23T14:27:08,284 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2] 2024-11-23T14:27:08,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on 3d5e64ff6f2c8eec1637e7e45ace6102 2024-11-23T14:27:08,285 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T14:27:08,285 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3d5e64ff6f2c8eec1637e7e45ace6102 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-23T14:27:08,288 DEBUG [Thread-2179 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T14:27:08,289 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6f7b1897123,35385,1732371902767, seqNum=-1] 2024-11-23T14:27:08,289 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T14:27:08,290 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32878, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T14:27:08,291 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2] 2024-11-23T14:27:08,313 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/A/4326dcf71488476781a63ed39c584cb6 is 50, key is test_row_0/A:col10/1732372028284/Put/seqid=0 2024-11-23T14:27:08,318 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:08,318 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:08,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32798 deadline: 1732372088311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:08,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32854 deadline: 1732372088310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:08,319 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:08,319 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:08,319 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:08,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32840 deadline: 1732372088312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:08,319 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:08,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:08,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:08,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:08,319 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:08,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32862 deadline: 1732372088316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:08,320 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:08,320 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:08,320 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:08,320 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:08,320 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:08,320 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:08,324 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:08,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32790 deadline: 1732372088317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:08,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:08,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:08,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:08,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742312_1488 (size=16681) 2024-11-23T14:27:08,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-23T14:27:08,429 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-23T14:27:08,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:08,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. as already flushing 2024-11-23T14:27:08,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:08,429 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:08,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:08,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:08,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:08,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32854 deadline: 1732372088430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:08,435 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:08,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:08,435 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:08,435 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:08,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32798 deadline: 1732372088430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:08,435 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:08,435 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:08,435 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:08,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:08,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32840 deadline: 1732372088430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:08,435 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:08,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:08,435 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:08,435 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:08,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32862 deadline: 1732372088430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:08,436 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:08,436 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:08,436 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:08,436 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:08,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32790 deadline: 1732372088430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:08,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:08,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:08,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:08,582 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-23T14:27:08,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:08,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. as already flushing 2024-11-23T14:27:08,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:08,582 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:08,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:08,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:08,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-23T14:27:08,643 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:08,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32854 deadline: 1732372088640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:08,643 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:08,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32798 deadline: 1732372088640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:08,643 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:08,643 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:08,643 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:08,643 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:08,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32840 deadline: 1732372088640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:08,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:08,644 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:08,644 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:08,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32862 deadline: 1732372088640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:08,644 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:08,644 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:08,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:08,644 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:08,644 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:08,644 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:08,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32790 deadline: 1732372088641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:08,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:08,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:08,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:08,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:08,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:08,728 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/A/4326dcf71488476781a63ed39c584cb6 2024-11-23T14:27:08,734 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-23T14:27:08,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:08,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. as already flushing 2024-11-23T14:27:08,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:08,735 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:08,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:08,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:08,750 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/B/5e21501f70d84a91a20ee58ddcad83cc is 50, key is test_row_0/B:col10/1732372028284/Put/seqid=0 2024-11-23T14:27:08,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742313_1489 (size=12001) 2024-11-23T14:27:08,887 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-23T14:27:08,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:08,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. as already flushing 2024-11-23T14:27:08,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:08,887 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:08,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:08,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:08,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-23T14:27:08,955 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:08,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32854 deadline: 1732372088950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:08,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:08,956 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:08,956 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:08,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:08,956 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:08,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32840 deadline: 1732372088950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:08,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32798 deadline: 1732372088950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:08,956 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:08,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:08,956 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:08,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:08,956 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:08,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32790 deadline: 1732372088951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:08,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:08,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:08,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:08,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32862 deadline: 1732372088951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:08,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:08,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:08,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:08,956 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:08,957 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:08,957 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:09,040 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-23T14:27:09,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:09,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. as already flushing 2024-11-23T14:27:09,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:09,040 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:09,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:09,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:09,155 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/B/5e21501f70d84a91a20ee58ddcad83cc 2024-11-23T14:27:09,173 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/C/916750a40ed6423789ec75c05b20021d is 50, key is test_row_0/C:col10/1732372028284/Put/seqid=0 2024-11-23T14:27:09,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742314_1490 (size=12001) 2024-11-23T14:27:09,191 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-23T14:27:09,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:09,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. as already flushing 2024-11-23T14:27:09,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:09,192 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:09,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:09,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:09,344 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-23T14:27:09,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:09,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. as already flushing 2024-11-23T14:27:09,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:09,344 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:09,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:09,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:09,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-23T14:27:09,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:09,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32854 deadline: 1732372089460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:09,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:09,465 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:09,465 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:09,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32840 deadline: 1732372089460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:09,465 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:09,465 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:09,465 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:09,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:09,465 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:09,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32790 deadline: 1732372089460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:09,465 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:09,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:09,465 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:09,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:09,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32862 deadline: 1732372089461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:09,466 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:09,466 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:09,466 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:09,473 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:09,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32798 deadline: 1732372089470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:09,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:09,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:09,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:09,496 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-23T14:27:09,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:09,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. as already flushing 2024-11-23T14:27:09,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:09,497 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:09,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:09,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:09,577 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/C/916750a40ed6423789ec75c05b20021d 2024-11-23T14:27:09,581 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/A/4326dcf71488476781a63ed39c584cb6 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/4326dcf71488476781a63ed39c584cb6 2024-11-23T14:27:09,584 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/4326dcf71488476781a63ed39c584cb6, entries=250, sequenceid=15, filesize=16.3 K 2024-11-23T14:27:09,584 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/B/5e21501f70d84a91a20ee58ddcad83cc as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/5e21501f70d84a91a20ee58ddcad83cc 2024-11-23T14:27:09,587 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/5e21501f70d84a91a20ee58ddcad83cc, entries=150, sequenceid=15, filesize=11.7 K 2024-11-23T14:27:09,587 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/C/916750a40ed6423789ec75c05b20021d as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/916750a40ed6423789ec75c05b20021d 2024-11-23T14:27:09,590 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/916750a40ed6423789ec75c05b20021d, entries=150, sequenceid=15, filesize=11.7 K 2024-11-23T14:27:09,591 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 3d5e64ff6f2c8eec1637e7e45ace6102 in 1305ms, sequenceid=15, compaction requested=false 2024-11-23T14:27:09,591 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-23T14:27:09,591 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3d5e64ff6f2c8eec1637e7e45ace6102: 2024-11-23T14:27:09,649 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-23T14:27:09,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:09,649 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 3d5e64ff6f2c8eec1637e7e45ace6102 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-23T14:27:09,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/A/2cd3002d22d04490bcf64f7600b3645c is 50, key is test_row_0/A:col10/1732372028310/Put/seqid=0 2024-11-23T14:27:09,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742315_1491 (size=12001) 2024-11-23T14:27:09,658 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/A/2cd3002d22d04490bcf64f7600b3645c 2024-11-23T14:27:09,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/B/39d879c2832b44408f2b7b843fb78c67 is 50, key is test_row_0/B:col10/1732372028310/Put/seqid=0 2024-11-23T14:27:09,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742316_1492 (size=12001) 2024-11-23T14:27:10,069 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/B/39d879c2832b44408f2b7b843fb78c67 2024-11-23T14:27:10,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/C/c22220c004c64c4aa36dbd9be665169f is 50, key is test_row_0/C:col10/1732372028310/Put/seqid=0 2024-11-23T14:27:10,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742317_1493 (size=12001) 2024-11-23T14:27:10,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-23T14:27:10,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on 3d5e64ff6f2c8eec1637e7e45ace6102 2024-11-23T14:27:10,475 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. as already flushing 2024-11-23T14:27:10,479 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/C/c22220c004c64c4aa36dbd9be665169f 2024-11-23T14:27:10,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/A/2cd3002d22d04490bcf64f7600b3645c as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/2cd3002d22d04490bcf64f7600b3645c 2024-11-23T14:27:10,491 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/2cd3002d22d04490bcf64f7600b3645c, entries=150, sequenceid=37, filesize=11.7 K 2024-11-23T14:27:10,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/B/39d879c2832b44408f2b7b843fb78c67 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/39d879c2832b44408f2b7b843fb78c67 2024-11-23T14:27:10,495 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/39d879c2832b44408f2b7b843fb78c67, entries=150, sequenceid=37, filesize=11.7 K 2024-11-23T14:27:10,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/C/c22220c004c64c4aa36dbd9be665169f as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/c22220c004c64c4aa36dbd9be665169f 2024-11-23T14:27:10,496 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:10,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32798 deadline: 1732372090490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:10,496 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to c6f7b1897123,35385,1732371902767 for 'test_row_0' in 3d5e64ff6f2c8eec1637e7e45ace6102 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-23T14:27:10,497 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:10,497 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:10,497 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:10,502 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/c22220c004c64c4aa36dbd9be665169f, entries=150, sequenceid=37, filesize=11.7 K 2024-11-23T14:27:10,503 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 3d5e64ff6f2c8eec1637e7e45ace6102 in 854ms, sequenceid=37, compaction requested=false 2024-11-23T14:27:10,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 3d5e64ff6f2c8eec1637e7e45ace6102: 2024-11-23T14:27:10,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:10,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-23T14:27:10,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-23T14:27:10,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on 3d5e64ff6f2c8eec1637e7e45ace6102 2024-11-23T14:27:10,504 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T14:27:10,505 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3d5e64ff6f2c8eec1637e7e45ace6102 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-23T14:27:10,506 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-23T14:27:10,506 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2280 sec 2024-11-23T14:27:10,508 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 2.2340 sec 2024-11-23T14:27:10,510 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/A/e64a4392a9434152bbbc862a3648d9c5 is 50, key is test_row_0/A:col10/1732372030504/Put/seqid=0 2024-11-23T14:27:10,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742318_1494 (size=12001) 2024-11-23T14:27:10,537 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:10,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32862 deadline: 1732372090535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:10,538 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:10,538 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:10,538 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:10,547 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:10,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32790 deadline: 1732372090536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:10,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:10,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:10,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:10,547 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:10,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32840 deadline: 1732372090536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:10,547 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:10,547 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:10,548 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:10,548 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:10,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32854 deadline: 1732372090537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:10,549 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:10,549 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:10,549 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:10,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:10,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32862 deadline: 1732372090640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:10,646 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:10,646 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:10,646 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:10,656 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:10,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32790 deadline: 1732372090650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:10,656 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:10,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:10,656 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:10,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:10,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32840 deadline: 1732372090650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:10,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:10,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32854 deadline: 1732372090651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:10,656 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:10,657 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:10,657 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:10,657 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:10,657 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:10,657 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:10,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:10,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32862 deadline: 1732372090850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:10,855 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:10,855 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:10,855 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:10,863 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-23T14:27:10,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:10,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32790 deadline: 1732372090860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:10,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:10,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:10,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:10,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32854 deadline: 1732372090860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:10,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:10,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:10,865 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:10,865 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:10,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32840 deadline: 1732372090861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:10,865 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:10,866 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:10,866 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:10,866 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:10,915 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/A/e64a4392a9434152bbbc862a3648d9c5 2024-11-23T14:27:10,920 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/B/7f63d5a736c143aaa83140b08517eda8 is 50, key is test_row_0/B:col10/1732372030504/Put/seqid=0 2024-11-23T14:27:10,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742319_1495 (size=12001) 2024-11-23T14:27:10,931 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/B/7f63d5a736c143aaa83140b08517eda8 2024-11-23T14:27:10,937 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/C/585e4a63cb474c81847372778bdf56b6 is 50, key is test_row_0/C:col10/1732372030504/Put/seqid=0 2024-11-23T14:27:10,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742320_1496 (size=12001) 2024-11-23T14:27:11,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:11,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32862 deadline: 1732372091160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:11,164 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:11,164 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:11,164 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:11,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:11,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32790 deadline: 1732372091170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:11,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:11,176 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:11,176 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:11,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32854 deadline: 1732372091170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:11,176 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:11,176 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:11,176 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:11,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:11,176 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:11,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32840 deadline: 1732372091170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:11,176 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:11,176 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:11,176 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:11,343 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/C/585e4a63cb474c81847372778bdf56b6 2024-11-23T14:27:11,347 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/A/e64a4392a9434152bbbc862a3648d9c5 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/e64a4392a9434152bbbc862a3648d9c5 2024-11-23T14:27:11,350 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/e64a4392a9434152bbbc862a3648d9c5, entries=150, sequenceid=54, filesize=11.7 K 2024-11-23T14:27:11,351 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/B/7f63d5a736c143aaa83140b08517eda8 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/7f63d5a736c143aaa83140b08517eda8 2024-11-23T14:27:11,354 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/7f63d5a736c143aaa83140b08517eda8, entries=150, sequenceid=54, filesize=11.7 K 2024-11-23T14:27:11,359 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/C/585e4a63cb474c81847372778bdf56b6 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/585e4a63cb474c81847372778bdf56b6 2024-11-23T14:27:11,364 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/585e4a63cb474c81847372778bdf56b6, entries=150, sequenceid=54, filesize=11.7 K 2024-11-23T14:27:11,365 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 3d5e64ff6f2c8eec1637e7e45ace6102 in 861ms, sequenceid=54, compaction requested=true 2024-11-23T14:27:11,365 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3d5e64ff6f2c8eec1637e7e45ace6102: 2024-11-23T14:27:11,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3d5e64ff6f2c8eec1637e7e45ace6102:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T14:27:11,365 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:27:11,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:27:11,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3d5e64ff6f2c8eec1637e7e45ace6102:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T14:27:11,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:27:11,365 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:27:11,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3d5e64ff6f2c8eec1637e7e45ace6102:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T14:27:11,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:27:11,369 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:27:11,369 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:27:11,369 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1541): 3d5e64ff6f2c8eec1637e7e45ace6102/A is initiating minor compaction (all files) 2024-11-23T14:27:11,369 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1541): 3d5e64ff6f2c8eec1637e7e45ace6102/B is initiating minor compaction (all files) 2024-11-23T14:27:11,369 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3d5e64ff6f2c8eec1637e7e45ace6102/B in TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:11,369 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3d5e64ff6f2c8eec1637e7e45ace6102/A in TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:11,369 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/4326dcf71488476781a63ed39c584cb6, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/2cd3002d22d04490bcf64f7600b3645c, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/e64a4392a9434152bbbc862a3648d9c5] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp, totalSize=39.7 K 2024-11-23T14:27:11,369 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/5e21501f70d84a91a20ee58ddcad83cc, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/39d879c2832b44408f2b7b843fb78c67, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/7f63d5a736c143aaa83140b08517eda8] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp, totalSize=35.2 K 2024-11-23T14:27:11,369 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4326dcf71488476781a63ed39c584cb6, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732372028280 2024-11-23T14:27:11,369 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e21501f70d84a91a20ee58ddcad83cc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732372028284 2024-11-23T14:27:11,370 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2cd3002d22d04490bcf64f7600b3645c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732372028308 2024-11-23T14:27:11,370 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 39d879c2832b44408f2b7b843fb78c67, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732372028308 2024-11-23T14:27:11,370 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting e64a4392a9434152bbbc862a3648d9c5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1732372030495 2024-11-23T14:27:11,370 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f63d5a736c143aaa83140b08517eda8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1732372030495 2024-11-23T14:27:11,377 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3d5e64ff6f2c8eec1637e7e45ace6102#B#compaction#416 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:27:11,377 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3d5e64ff6f2c8eec1637e7e45ace6102#A#compaction#417 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:27:11,377 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/B/cab0515b1ee241e8934188a6362891a0 is 50, key is test_row_0/B:col10/1732372030504/Put/seqid=0 2024-11-23T14:27:11,377 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/A/405d6d89a67d4480905356ee2054dc33 is 50, key is test_row_0/A:col10/1732372030504/Put/seqid=0 2024-11-23T14:27:11,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742321_1497 (size=12104) 2024-11-23T14:27:11,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742322_1498 (size=12104) 2024-11-23T14:27:11,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on 3d5e64ff6f2c8eec1637e7e45ace6102 2024-11-23T14:27:11,673 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T14:27:11,673 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3d5e64ff6f2c8eec1637e7e45ace6102 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-23T14:27:11,677 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/A/be523536339b46609d06d999bb0f64bf is 50, key is test_row_1/A:col10/1732372031672/Put/seqid=0 2024-11-23T14:27:11,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742323_1499 (size=11997) 2024-11-23T14:27:11,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:11,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32862 deadline: 1732372091700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:11,707 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:11,707 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:11,707 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:11,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:11,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32854 deadline: 1732372091700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:11,707 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:11,707 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:11,707 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:11,708 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:11,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32840 deadline: 1732372091706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:11,708 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:11,708 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:11,708 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:11,708 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:11,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32790 deadline: 1732372091707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:11,709 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:11,709 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:11,709 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:11,789 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/B/cab0515b1ee241e8934188a6362891a0 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/cab0515b1ee241e8934188a6362891a0 2024-11-23T14:27:11,793 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3d5e64ff6f2c8eec1637e7e45ace6102/B of 3d5e64ff6f2c8eec1637e7e45ace6102 into cab0515b1ee241e8934188a6362891a0(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:27:11,793 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3d5e64ff6f2c8eec1637e7e45ace6102: 2024-11-23T14:27:11,793 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., storeName=3d5e64ff6f2c8eec1637e7e45ace6102/B, priority=13, startTime=1732372031365; duration=0sec 2024-11-23T14:27:11,793 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:27:11,793 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/A/405d6d89a67d4480905356ee2054dc33 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/405d6d89a67d4480905356ee2054dc33 2024-11-23T14:27:11,793 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3d5e64ff6f2c8eec1637e7e45ace6102:B 2024-11-23T14:27:11,793 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:27:11,794 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:27:11,794 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1541): 3d5e64ff6f2c8eec1637e7e45ace6102/C is initiating minor compaction (all files) 2024-11-23T14:27:11,794 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3d5e64ff6f2c8eec1637e7e45ace6102/C in TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:11,794 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/916750a40ed6423789ec75c05b20021d, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/c22220c004c64c4aa36dbd9be665169f, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/585e4a63cb474c81847372778bdf56b6] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp, totalSize=35.2 K 2024-11-23T14:27:11,795 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 916750a40ed6423789ec75c05b20021d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732372028284 2024-11-23T14:27:11,795 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting c22220c004c64c4aa36dbd9be665169f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732372028308 2024-11-23T14:27:11,795 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 585e4a63cb474c81847372778bdf56b6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1732372030495 2024-11-23T14:27:11,797 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3d5e64ff6f2c8eec1637e7e45ace6102/A of 3d5e64ff6f2c8eec1637e7e45ace6102 into 405d6d89a67d4480905356ee2054dc33(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:27:11,797 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3d5e64ff6f2c8eec1637e7e45ace6102: 2024-11-23T14:27:11,797 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., storeName=3d5e64ff6f2c8eec1637e7e45ace6102/A, priority=13, startTime=1732372031365; duration=0sec 2024-11-23T14:27:11,797 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:27:11,797 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3d5e64ff6f2c8eec1637e7e45ace6102:A 2024-11-23T14:27:11,805 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3d5e64ff6f2c8eec1637e7e45ace6102#C#compaction#419 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:27:11,806 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/C/72946c50ebc7494d90b6b42f7d7736d2 is 50, key is test_row_0/C:col10/1732372030504/Put/seqid=0 2024-11-23T14:27:11,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742324_1500 (size=12104) 2024-11-23T14:27:11,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:11,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32854 deadline: 1732372091810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:11,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:11,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32862 deadline: 1732372091810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:11,816 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:11,816 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:11,816 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:11,816 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:11,816 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:11,816 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:11,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:11,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32840 deadline: 1732372091820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:11,826 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:11,826 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:11,826 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:11,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:11,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32790 deadline: 1732372091820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:11,827 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:11,827 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:11,827 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:12,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:12,025 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:12,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32854 deadline: 1732372092020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:12,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32862 deadline: 1732372092020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:12,025 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:12,025 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:12,025 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:12,025 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:12,025 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:12,025 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:12,032 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:12,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32840 deadline: 1732372092030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:12,032 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:12,032 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:12,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32790 deadline: 1732372092030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:12,033 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:12,033 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:12,033 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:12,033 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:12,033 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:12,084 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/A/be523536339b46609d06d999bb0f64bf 2024-11-23T14:27:12,090 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/B/a4ddab881bdb4174bd1d6020c31b8d1f is 50, key is test_row_1/B:col10/1732372031672/Put/seqid=0 2024-11-23T14:27:12,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742325_1501 (size=9657) 2024-11-23T14:27:12,217 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/C/72946c50ebc7494d90b6b42f7d7736d2 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/72946c50ebc7494d90b6b42f7d7736d2 2024-11-23T14:27:12,221 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3d5e64ff6f2c8eec1637e7e45ace6102/C of 3d5e64ff6f2c8eec1637e7e45ace6102 into 72946c50ebc7494d90b6b42f7d7736d2(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:27:12,221 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3d5e64ff6f2c8eec1637e7e45ace6102: 2024-11-23T14:27:12,221 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., storeName=3d5e64ff6f2c8eec1637e7e45ace6102/C, priority=13, startTime=1732372031365; duration=0sec 2024-11-23T14:27:12,221 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:27:12,221 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3d5e64ff6f2c8eec1637e7e45ace6102:C 2024-11-23T14:27:12,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:12,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32854 deadline: 1732372092330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:12,335 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:12,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:12,335 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:12,335 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:12,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32862 deadline: 1732372092330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:12,335 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:12,335 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:12,335 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:12,345 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:12,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32840 deadline: 1732372092340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:12,345 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:12,345 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:12,345 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:12,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32790 deadline: 1732372092340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:12,345 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:12,345 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:12,345 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:12,345 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:12,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-23T14:27:12,430 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T14:27:12,430 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-23T14:27:12,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-23T14:27:12,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-23T14:27:12,432 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T14:27:12,432 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T14:27:12,432 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T14:27:12,493 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/B/a4ddab881bdb4174bd1d6020c31b8d1f 2024-11-23T14:27:12,499 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/C/7e564de2c12a41468568aff8d64c2794 is 50, key is test_row_1/C:col10/1732372031672/Put/seqid=0 2024-11-23T14:27:12,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742326_1502 (size=9657) 2024-11-23T14:27:12,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:12,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32798 deadline: 1732372092510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:12,512 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to c6f7b1897123,35385,1732371902767 for 'test_row_0' in 3d5e64ff6f2c8eec1637e7e45ace6102 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-23T14:27:12,513 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:12,513 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:12,513 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:12,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-23T14:27:12,584 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-23T14:27:12,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:12,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. as already flushing 2024-11-23T14:27:12,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:12,584 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:12,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:12,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:12,737 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-23T14:27:12,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:12,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. as already flushing 2024-11-23T14:27:12,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:12,737 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:12,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:12,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:12,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-23T14:27:12,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:12,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32854 deadline: 1732372092840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:12,841 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:12,841 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:12,841 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:12,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:12,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32862 deadline: 1732372092840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:12,841 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:12,841 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:12,841 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:12,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:12,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32840 deadline: 1732372092850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:12,856 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:12,856 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:12,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:12,857 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:12,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32790 deadline: 1732372092850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:12,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:12,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:12,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:12,889 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-23T14:27:12,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:12,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. as already flushing 2024-11-23T14:27:12,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:12,890 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:12,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:12,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:12,902 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/C/7e564de2c12a41468568aff8d64c2794 2024-11-23T14:27:12,906 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/A/be523536339b46609d06d999bb0f64bf as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/be523536339b46609d06d999bb0f64bf 2024-11-23T14:27:12,908 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/be523536339b46609d06d999bb0f64bf, entries=150, sequenceid=74, filesize=11.7 K 2024-11-23T14:27:12,909 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/B/a4ddab881bdb4174bd1d6020c31b8d1f as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/a4ddab881bdb4174bd1d6020c31b8d1f 2024-11-23T14:27:12,911 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/a4ddab881bdb4174bd1d6020c31b8d1f, entries=100, sequenceid=74, filesize=9.4 K 2024-11-23T14:27:12,912 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/C/7e564de2c12a41468568aff8d64c2794 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/7e564de2c12a41468568aff8d64c2794 2024-11-23T14:27:12,914 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/7e564de2c12a41468568aff8d64c2794, entries=100, sequenceid=74, filesize=9.4 K 2024-11-23T14:27:12,915 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=93.93 KB/96180 for 3d5e64ff6f2c8eec1637e7e45ace6102 in 1242ms, sequenceid=74, compaction requested=false 2024-11-23T14:27:12,915 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3d5e64ff6f2c8eec1637e7e45ace6102: 2024-11-23T14:27:13,042 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-23T14:27:13,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:13,042 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 3d5e64ff6f2c8eec1637e7e45ace6102 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-23T14:27:13,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/A/b36eb1221c1a43ca856d4482b40ef868 is 50, key is test_row_0/A:col10/1732372031689/Put/seqid=0 2024-11-23T14:27:13,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742327_1503 (size=12001) 2024-11-23T14:27:13,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-23T14:27:13,451 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/A/b36eb1221c1a43ca856d4482b40ef868 2024-11-23T14:27:13,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/B/9de94979b4ca42ab801ce8cdae9fa37c is 50, key is test_row_0/B:col10/1732372031689/Put/seqid=0 2024-11-23T14:27:13,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742328_1504 (size=12001) 2024-11-23T14:27:13,461 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/B/9de94979b4ca42ab801ce8cdae9fa37c 2024-11-23T14:27:13,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/C/d7dc4004cbc046acb1af2a6c38e0367a is 50, key is test_row_0/C:col10/1732372031689/Put/seqid=0 2024-11-23T14:27:13,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742329_1505 (size=12001) 2024-11-23T14:27:13,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-23T14:27:13,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on 3d5e64ff6f2c8eec1637e7e45ace6102 2024-11-23T14:27:13,852 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. as already flushing 2024-11-23T14:27:13,870 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/C/d7dc4004cbc046acb1af2a6c38e0367a 2024-11-23T14:27:13,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/A/b36eb1221c1a43ca856d4482b40ef868 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/b36eb1221c1a43ca856d4482b40ef868 2024-11-23T14:27:13,876 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/b36eb1221c1a43ca856d4482b40ef868, entries=150, sequenceid=94, filesize=11.7 K 2024-11-23T14:27:13,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/B/9de94979b4ca42ab801ce8cdae9fa37c as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/9de94979b4ca42ab801ce8cdae9fa37c 2024-11-23T14:27:13,880 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/9de94979b4ca42ab801ce8cdae9fa37c, entries=150, sequenceid=94, filesize=11.7 K 2024-11-23T14:27:13,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/C/d7dc4004cbc046acb1af2a6c38e0367a as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/d7dc4004cbc046acb1af2a6c38e0367a 2024-11-23T14:27:13,883 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/d7dc4004cbc046acb1af2a6c38e0367a, entries=150, sequenceid=94, filesize=11.7 K 2024-11-23T14:27:13,884 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=100.63 KB/103050 for 3d5e64ff6f2c8eec1637e7e45ace6102 in 842ms, sequenceid=94, compaction requested=true 2024-11-23T14:27:13,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 3d5e64ff6f2c8eec1637e7e45ace6102: 2024-11-23T14:27:13,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:13,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-23T14:27:13,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-23T14:27:13,886 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-23T14:27:13,886 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4530 sec 2024-11-23T14:27:13,888 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 1.4570 sec 2024-11-23T14:27:13,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on 3d5e64ff6f2c8eec1637e7e45ace6102 2024-11-23T14:27:13,889 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T14:27:13,889 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3d5e64ff6f2c8eec1637e7e45ace6102 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-23T14:27:13,892 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/A/aac3f99378fc48bab42bb84d214f64a8 is 50, key is test_row_0/A:col10/1732372033881/Put/seqid=0 2024-11-23T14:27:13,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742330_1506 (size=19021) 2024-11-23T14:27:13,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:13,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32840 deadline: 1732372093920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:13,928 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:13,928 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:13,928 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:13,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:13,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32790 deadline: 1732372093920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:13,930 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:13,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32854 deadline: 1732372093921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:13,930 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:13,930 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:13,930 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:13,930 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:13,930 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:13,930 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:13,932 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:13,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32862 deadline: 1732372093928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:13,932 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:13,932 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:13,932 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:14,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:14,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32840 deadline: 1732372094040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:14,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:14,048 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:14,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32790 deadline: 1732372094040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:14,048 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:14,048 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:14,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:14,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:14,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:14,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32854 deadline: 1732372094041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:14,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:14,049 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:14,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:14,049 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:14,049 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:14,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32862 deadline: 1732372094041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:14,049 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:14,049 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:14,049 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:14,254 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:14,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32840 deadline: 1732372094250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:14,254 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:14,254 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:14,254 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:14,262 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:14,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32790 deadline: 1732372094259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:14,263 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:14,263 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:14,263 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:14,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32854 deadline: 1732372094260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:14,263 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:14,263 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:14,263 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:14,263 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:14,263 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:14,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32862 deadline: 1732372094260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:14,263 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:14,263 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:14,263 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:14,296 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/A/aac3f99378fc48bab42bb84d214f64a8 2024-11-23T14:27:14,302 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/B/12a20af8407545f8a808daf3c3a5941e is 50, key is test_row_0/B:col10/1732372033881/Put/seqid=0 2024-11-23T14:27:14,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742331_1507 (size=12001) 2024-11-23T14:27:14,307 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/B/12a20af8407545f8a808daf3c3a5941e 2024-11-23T14:27:14,313 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/C/09456c5655fc4cf999eec817cf5875a7 is 50, key is test_row_0/C:col10/1732372033881/Put/seqid=0 2024-11-23T14:27:14,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742332_1508 (size=12001) 2024-11-23T14:27:14,565 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:14,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32840 deadline: 1732372094560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:14,565 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:14,565 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:14,565 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:14,573 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:14,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32790 deadline: 1732372094570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:14,574 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:14,574 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:14,574 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:14,574 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:14,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32854 deadline: 1732372094570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:14,574 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:14,574 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:14,574 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:14,574 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:14,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32862 deadline: 1732372094570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:14,574 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:14,574 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:14,574 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:14,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-23T14:27:14,580 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T14:27:14,580 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-23T14:27:14,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-23T14:27:14,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-23T14:27:14,582 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T14:27:14,582 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T14:27:14,582 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T14:27:14,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-23T14:27:14,723 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/C/09456c5655fc4cf999eec817cf5875a7 2024-11-23T14:27:14,727 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/A/aac3f99378fc48bab42bb84d214f64a8 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/aac3f99378fc48bab42bb84d214f64a8 2024-11-23T14:27:14,730 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/aac3f99378fc48bab42bb84d214f64a8, entries=300, sequenceid=113, filesize=18.6 K 2024-11-23T14:27:14,730 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/B/12a20af8407545f8a808daf3c3a5941e as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/12a20af8407545f8a808daf3c3a5941e 2024-11-23T14:27:14,733 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/12a20af8407545f8a808daf3c3a5941e, entries=150, sequenceid=113, filesize=11.7 K 2024-11-23T14:27:14,734 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/C/09456c5655fc4cf999eec817cf5875a7 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/09456c5655fc4cf999eec817cf5875a7 2024-11-23T14:27:14,734 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-23T14:27:14,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:14,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. as already flushing 2024-11-23T14:27:14,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:14,735 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:14,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:14,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:14,739 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/09456c5655fc4cf999eec817cf5875a7, entries=150, sequenceid=113, filesize=11.7 K 2024-11-23T14:27:14,740 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 3d5e64ff6f2c8eec1637e7e45ace6102 in 851ms, sequenceid=113, compaction requested=true 2024-11-23T14:27:14,740 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3d5e64ff6f2c8eec1637e7e45ace6102: 2024-11-23T14:27:14,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3d5e64ff6f2c8eec1637e7e45ace6102:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T14:27:14,740 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T14:27:14,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:27:14,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3d5e64ff6f2c8eec1637e7e45ace6102:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T14:27:14,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:27:14,740 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T14:27:14,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3d5e64ff6f2c8eec1637e7e45ace6102:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T14:27:14,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:27:14,741 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55123 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T14:27:14,741 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45763 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T14:27:14,741 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1541): 3d5e64ff6f2c8eec1637e7e45ace6102/B is initiating minor compaction (all files) 2024-11-23T14:27:14,741 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1541): 3d5e64ff6f2c8eec1637e7e45ace6102/A is initiating minor compaction (all files) 2024-11-23T14:27:14,741 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3d5e64ff6f2c8eec1637e7e45ace6102/A in TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:14,741 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/405d6d89a67d4480905356ee2054dc33, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/be523536339b46609d06d999bb0f64bf, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/b36eb1221c1a43ca856d4482b40ef868, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/aac3f99378fc48bab42bb84d214f64a8] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp, totalSize=53.8 K 2024-11-23T14:27:14,741 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3d5e64ff6f2c8eec1637e7e45ace6102/B in TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:14,742 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/cab0515b1ee241e8934188a6362891a0, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/a4ddab881bdb4174bd1d6020c31b8d1f, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/9de94979b4ca42ab801ce8cdae9fa37c, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/12a20af8407545f8a808daf3c3a5941e] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp, totalSize=44.7 K 2024-11-23T14:27:14,742 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 405d6d89a67d4480905356ee2054dc33, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1732372030495 2024-11-23T14:27:14,742 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting cab0515b1ee241e8934188a6362891a0, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1732372030495 2024-11-23T14:27:14,742 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting be523536339b46609d06d999bb0f64bf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1732372030534 2024-11-23T14:27:14,742 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting a4ddab881bdb4174bd1d6020c31b8d1f, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1732372030534 2024-11-23T14:27:14,742 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting b36eb1221c1a43ca856d4482b40ef868, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732372031689 2024-11-23T14:27:14,742 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting aac3f99378fc48bab42bb84d214f64a8, keycount=300, bloomtype=ROW, size=18.6 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1732372033862 2024-11-23T14:27:14,742 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 9de94979b4ca42ab801ce8cdae9fa37c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732372031689 2024-11-23T14:27:14,742 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 12a20af8407545f8a808daf3c3a5941e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1732372033881 2024-11-23T14:27:14,748 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3d5e64ff6f2c8eec1637e7e45ace6102#B#compaction#428 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:27:14,748 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/B/4ecf371c663348a3bef6a539797ed0e2 is 50, key is test_row_0/B:col10/1732372033881/Put/seqid=0 2024-11-23T14:27:14,751 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3d5e64ff6f2c8eec1637e7e45ace6102#A#compaction#429 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:27:14,751 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/A/688a7150665d482d9513a16f3f0835c9 is 50, key is test_row_0/A:col10/1732372033881/Put/seqid=0 2024-11-23T14:27:14,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742333_1509 (size=12241) 2024-11-23T14:27:14,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742334_1510 (size=12241) 2024-11-23T14:27:14,887 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-23T14:27:14,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:14,887 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing 3d5e64ff6f2c8eec1637e7e45ace6102 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-23T14:27:14,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/A/735cdb09a36b4058b8aac96bc5149d10 is 50, key is test_row_0/A:col10/1732372033906/Put/seqid=0 2024-11-23T14:27:14,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-23T14:27:14,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742335_1511 (size=12051) 2024-11-23T14:27:15,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on 3d5e64ff6f2c8eec1637e7e45ace6102 2024-11-23T14:27:15,074 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. as already flushing 2024-11-23T14:27:15,126 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:15,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32862 deadline: 1732372095116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:15,126 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:15,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32840 deadline: 1732372095116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:15,126 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:15,126 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:15,126 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:15,126 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:15,126 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:15,126 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:15,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32790 deadline: 1732372095118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:15,126 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:15,126 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:15,126 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:15,126 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:15,131 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:15,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32854 deadline: 1732372095121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:15,132 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:15,132 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:15,132 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:15,156 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/B/4ecf371c663348a3bef6a539797ed0e2 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/4ecf371c663348a3bef6a539797ed0e2 2024-11-23T14:27:15,160 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3d5e64ff6f2c8eec1637e7e45ace6102/B of 3d5e64ff6f2c8eec1637e7e45ace6102 into 4ecf371c663348a3bef6a539797ed0e2(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:27:15,160 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3d5e64ff6f2c8eec1637e7e45ace6102: 2024-11-23T14:27:15,160 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., storeName=3d5e64ff6f2c8eec1637e7e45ace6102/B, priority=12, startTime=1732372034740; duration=0sec 2024-11-23T14:27:15,160 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:27:15,160 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3d5e64ff6f2c8eec1637e7e45ace6102:B 2024-11-23T14:27:15,160 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T14:27:15,161 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45763 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T14:27:15,161 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1541): 3d5e64ff6f2c8eec1637e7e45ace6102/C is initiating minor compaction (all files) 2024-11-23T14:27:15,161 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3d5e64ff6f2c8eec1637e7e45ace6102/C in TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:15,161 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/72946c50ebc7494d90b6b42f7d7736d2, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/7e564de2c12a41468568aff8d64c2794, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/d7dc4004cbc046acb1af2a6c38e0367a, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/09456c5655fc4cf999eec817cf5875a7] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp, totalSize=44.7 K 2024-11-23T14:27:15,161 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 72946c50ebc7494d90b6b42f7d7736d2, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1732372030495 2024-11-23T14:27:15,161 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/A/688a7150665d482d9513a16f3f0835c9 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/688a7150665d482d9513a16f3f0835c9 2024-11-23T14:27:15,161 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e564de2c12a41468568aff8d64c2794, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1732372030534 2024-11-23T14:27:15,162 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting d7dc4004cbc046acb1af2a6c38e0367a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732372031689 2024-11-23T14:27:15,162 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 09456c5655fc4cf999eec817cf5875a7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1732372033881 2024-11-23T14:27:15,165 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3d5e64ff6f2c8eec1637e7e45ace6102/A of 3d5e64ff6f2c8eec1637e7e45ace6102 into 688a7150665d482d9513a16f3f0835c9(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:27:15,165 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3d5e64ff6f2c8eec1637e7e45ace6102: 2024-11-23T14:27:15,165 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., storeName=3d5e64ff6f2c8eec1637e7e45ace6102/A, priority=12, startTime=1732372034740; duration=0sec 2024-11-23T14:27:15,165 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:27:15,165 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3d5e64ff6f2c8eec1637e7e45ace6102:A 2024-11-23T14:27:15,168 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3d5e64ff6f2c8eec1637e7e45ace6102#C#compaction#431 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:27:15,168 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/C/58cf4f7bb37f449588942376ef735ccd is 50, key is test_row_0/C:col10/1732372033881/Put/seqid=0 2024-11-23T14:27:15,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742336_1512 (size=12241) 2024-11-23T14:27:15,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-23T14:27:15,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:15,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32862 deadline: 1732372095230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:15,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:15,235 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:15,235 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:15,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32840 deadline: 1732372095230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:15,235 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:15,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:15,235 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:15,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32790 deadline: 1732372095230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:15,236 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:15,236 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:15,236 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:15,236 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:15,236 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:15,243 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:15,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32854 deadline: 1732372095240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:15,244 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:15,244 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:15,244 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:15,302 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/A/735cdb09a36b4058b8aac96bc5149d10 2024-11-23T14:27:15,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/B/b0503383ed8644e78aeb8dcb01046183 is 50, key is test_row_0/B:col10/1732372033906/Put/seqid=0 2024-11-23T14:27:15,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742337_1513 (size=12051) 2024-11-23T14:27:15,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:15,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32862 deadline: 1732372095440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:15,442 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:15,442 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:15,442 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:15,447 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:15,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32840 deadline: 1732372095440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:15,447 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:15,447 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:15,447 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:15,447 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:15,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32790 deadline: 1732372095440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:15,447 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:15,447 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:15,447 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:15,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:15,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32854 deadline: 1732372095450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:15,456 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:15,456 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:15,456 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:15,636 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/C/58cf4f7bb37f449588942376ef735ccd as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/58cf4f7bb37f449588942376ef735ccd 2024-11-23T14:27:15,639 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3d5e64ff6f2c8eec1637e7e45ace6102/C of 3d5e64ff6f2c8eec1637e7e45ace6102 into 58cf4f7bb37f449588942376ef735ccd(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:27:15,639 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3d5e64ff6f2c8eec1637e7e45ace6102: 2024-11-23T14:27:15,640 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., storeName=3d5e64ff6f2c8eec1637e7e45ace6102/C, priority=12, startTime=1732372034740; duration=0sec 2024-11-23T14:27:15,640 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:27:15,640 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3d5e64ff6f2c8eec1637e7e45ace6102:C 2024-11-23T14:27:15,713 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/B/b0503383ed8644e78aeb8dcb01046183 2024-11-23T14:27:15,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/C/b552ac49af494df8ad9e95934a5a10f0 is 50, key is test_row_0/C:col10/1732372033906/Put/seqid=0 2024-11-23T14:27:15,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-23T14:27:15,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742338_1514 (size=12051) 2024-11-23T14:27:15,755 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:15,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32862 deadline: 1732372095750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:15,756 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:15,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32840 deadline: 1732372095750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:15,756 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:15,756 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:15,756 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:15,756 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:15,756 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:15,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32790 deadline: 1732372095750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:15,756 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:15,756 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:15,756 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:15,756 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:15,756 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:15,765 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:15,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32854 deadline: 1732372095760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:15,766 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:15,766 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:15,766 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:16,124 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/C/b552ac49af494df8ad9e95934a5a10f0 2024-11-23T14:27:16,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/A/735cdb09a36b4058b8aac96bc5149d10 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/735cdb09a36b4058b8aac96bc5149d10 2024-11-23T14:27:16,130 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/735cdb09a36b4058b8aac96bc5149d10, entries=150, sequenceid=130, filesize=11.8 K 2024-11-23T14:27:16,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/B/b0503383ed8644e78aeb8dcb01046183 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/b0503383ed8644e78aeb8dcb01046183 2024-11-23T14:27:16,136 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/b0503383ed8644e78aeb8dcb01046183, entries=150, sequenceid=130, filesize=11.8 K 2024-11-23T14:27:16,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/C/b552ac49af494df8ad9e95934a5a10f0 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/b552ac49af494df8ad9e95934a5a10f0 2024-11-23T14:27:16,141 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/b552ac49af494df8ad9e95934a5a10f0, entries=150, sequenceid=130, filesize=11.8 K 2024-11-23T14:27:16,142 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 3d5e64ff6f2c8eec1637e7e45ace6102 in 1255ms, sequenceid=130, compaction requested=false 2024-11-23T14:27:16,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for 3d5e64ff6f2c8eec1637e7e45ace6102: 2024-11-23T14:27:16,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:16,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-23T14:27:16,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-23T14:27:16,145 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-23T14:27:16,145 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5610 sec 2024-11-23T14:27:16,146 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 1.5650 sec 2024-11-23T14:27:16,265 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T14:27:16,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on 3d5e64ff6f2c8eec1637e7e45ace6102 2024-11-23T14:27:16,265 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3d5e64ff6f2c8eec1637e7e45ace6102 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-23T14:27:16,269 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/A/93b311ee242c4765b199e56aabe970c3 is 50, key is test_row_0/A:col10/1732372035120/Put/seqid=0 2024-11-23T14:27:16,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742339_1515 (size=14541) 2024-11-23T14:27:16,298 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:16,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32790 deadline: 1732372096295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:16,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:16,298 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:16,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:16,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:16,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32854 deadline: 1732372096296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:16,298 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:16,298 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:16,298 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:16,304 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:16,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32840 deadline: 1732372096297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:16,304 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:16,304 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:16,304 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:16,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32862 deadline: 1732372096298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:16,304 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:16,304 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:16,304 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:16,304 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:16,406 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:16,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32790 deadline: 1732372096400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:16,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:16,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:16,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:16,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32854 deadline: 1732372096401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:16,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:16,407 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:16,407 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:16,407 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:16,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:16,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32840 deadline: 1732372096410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:16,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:16,415 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:16,415 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:16,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32862 deadline: 1732372096410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:16,416 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:16,416 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:16,416 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:16,416 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:16,543 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:16,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32798 deadline: 1732372096540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:16,543 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to c6f7b1897123,35385,1732371902767 for 'test_row_0' in 3d5e64ff6f2c8eec1637e7e45ace6102 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8232 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-23T14:27:16,544 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:16,544 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:16,544 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:16,613 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:16,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32790 deadline: 1732372096610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:16,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:16,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:16,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32854 deadline: 1732372096610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:16,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:16,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:16,614 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:16,614 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:16,614 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:16,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:16,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32840 deadline: 1732372096620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:16,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:16,625 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:16,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32862 deadline: 1732372096620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:16,625 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:16,625 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:16,625 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:16,625 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:16,625 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:16,673 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/A/93b311ee242c4765b199e56aabe970c3 2024-11-23T14:27:16,679 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/B/24817314092444f687176877654e99c4 is 50, key is test_row_0/B:col10/1732372035120/Put/seqid=0 2024-11-23T14:27:16,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742340_1516 (size=12151) 2024-11-23T14:27:16,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-23T14:27:16,730 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T14:27:16,731 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-23T14:27:16,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-23T14:27:16,732 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T14:27:16,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-23T14:27:16,733 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T14:27:16,733 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T14:27:16,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-23T14:27:16,885 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-23T14:27:16,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:16,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. as already flushing 2024-11-23T14:27:16,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:16,885 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:16,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:16,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:16,924 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:16,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32790 deadline: 1732372096920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:16,924 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:16,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:16,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:16,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32854 deadline: 1732372096920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:16,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:16,925 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:16,925 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:16,925 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:16,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:16,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32840 deadline: 1732372096930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:16,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:16,933 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:16,933 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:16,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32862 deadline: 1732372096930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:16,934 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:16,934 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:16,934 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:16,934 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:17,037 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-23T14:27:17,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:17,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. as already flushing 2024-11-23T14:27:17,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:17,038 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:17,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:17,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:17,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-23T14:27:17,083 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/B/24817314092444f687176877654e99c4 2024-11-23T14:27:17,089 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/C/8dbea164d23e4c8caa6213550b12bd6f is 50, key is test_row_0/C:col10/1732372035120/Put/seqid=0 2024-11-23T14:27:17,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742341_1517 (size=12151) 2024-11-23T14:27:17,190 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-23T14:27:17,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:17,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. as already flushing 2024-11-23T14:27:17,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:17,190 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:17,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:17,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:17,342 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-23T14:27:17,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:17,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. as already flushing 2024-11-23T14:27:17,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:17,343 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:17,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:17,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:17,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-23T14:27:17,436 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:17,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32790 deadline: 1732372097430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:17,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:17,436 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:17,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:17,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:17,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32854 deadline: 1732372097430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:17,437 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:17,437 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:17,437 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:17,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:17,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32840 deadline: 1732372097440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:17,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:17,446 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:17,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32862 deadline: 1732372097440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:17,446 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:17,446 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:17,446 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:17,446 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:17,446 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:17,492 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/C/8dbea164d23e4c8caa6213550b12bd6f 2024-11-23T14:27:17,495 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-23T14:27:17,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:17,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. as already flushing 2024-11-23T14:27:17,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:17,495 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:17,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:17,496 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/A/93b311ee242c4765b199e56aabe970c3 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/93b311ee242c4765b199e56aabe970c3 2024-11-23T14:27:17,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:17,500 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/93b311ee242c4765b199e56aabe970c3, entries=200, sequenceid=153, filesize=14.2 K 2024-11-23T14:27:17,501 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/B/24817314092444f687176877654e99c4 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/24817314092444f687176877654e99c4 2024-11-23T14:27:17,503 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/24817314092444f687176877654e99c4, entries=150, sequenceid=153, filesize=11.9 K 2024-11-23T14:27:17,504 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/C/8dbea164d23e4c8caa6213550b12bd6f as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/8dbea164d23e4c8caa6213550b12bd6f 2024-11-23T14:27:17,506 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/8dbea164d23e4c8caa6213550b12bd6f, entries=150, sequenceid=153, filesize=11.9 K 2024-11-23T14:27:17,507 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 3d5e64ff6f2c8eec1637e7e45ace6102 in 1242ms, sequenceid=153, compaction requested=true 2024-11-23T14:27:17,507 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3d5e64ff6f2c8eec1637e7e45ace6102: 2024-11-23T14:27:17,507 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3d5e64ff6f2c8eec1637e7e45ace6102:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T14:27:17,507 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:27:17,507 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:27:17,507 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3d5e64ff6f2c8eec1637e7e45ace6102:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T14:27:17,507 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:27:17,507 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3d5e64ff6f2c8eec1637e7e45ace6102:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T14:27:17,507 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:27:17,507 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:27:17,508 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38833 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:27:17,508 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36443 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:27:17,508 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1541): 3d5e64ff6f2c8eec1637e7e45ace6102/A is initiating minor compaction (all files) 2024-11-23T14:27:17,508 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1541): 3d5e64ff6f2c8eec1637e7e45ace6102/B is initiating minor compaction (all files) 2024-11-23T14:27:17,508 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3d5e64ff6f2c8eec1637e7e45ace6102/B in TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:17,508 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3d5e64ff6f2c8eec1637e7e45ace6102/A in TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:17,508 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/688a7150665d482d9513a16f3f0835c9, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/735cdb09a36b4058b8aac96bc5149d10, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/93b311ee242c4765b199e56aabe970c3] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp, totalSize=37.9 K 2024-11-23T14:27:17,508 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/4ecf371c663348a3bef6a539797ed0e2, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/b0503383ed8644e78aeb8dcb01046183, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/24817314092444f687176877654e99c4] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp, totalSize=35.6 K 2024-11-23T14:27:17,508 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 688a7150665d482d9513a16f3f0835c9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1732372033881 2024-11-23T14:27:17,508 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ecf371c663348a3bef6a539797ed0e2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1732372033881 2024-11-23T14:27:17,509 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 735cdb09a36b4058b8aac96bc5149d10, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1732372033906 2024-11-23T14:27:17,509 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting b0503383ed8644e78aeb8dcb01046183, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1732372033906 2024-11-23T14:27:17,509 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 93b311ee242c4765b199e56aabe970c3, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1732372035104 2024-11-23T14:27:17,509 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 24817314092444f687176877654e99c4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1732372035117 2024-11-23T14:27:17,527 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3d5e64ff6f2c8eec1637e7e45ace6102#B#compaction#437 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:27:17,527 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3d5e64ff6f2c8eec1637e7e45ace6102#A#compaction#438 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:27:17,528 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/B/c486d1ec51ad46579fc29e4a4cbe47b8 is 50, key is test_row_0/B:col10/1732372035120/Put/seqid=0 2024-11-23T14:27:17,528 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/A/63e9792bae4341279862609258c1be04 is 50, key is test_row_0/A:col10/1732372035120/Put/seqid=0 2024-11-23T14:27:17,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742342_1518 (size=12493) 2024-11-23T14:27:17,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742343_1519 (size=12493) 2024-11-23T14:27:17,647 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-23T14:27:17,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:17,648 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing 3d5e64ff6f2c8eec1637e7e45ace6102 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-23T14:27:17,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/A/24801b89b8a742adbaf0d5f240dcb832 is 50, key is test_row_0/A:col10/1732372036294/Put/seqid=0 2024-11-23T14:27:17,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742344_1520 (size=12151) 2024-11-23T14:27:17,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-23T14:27:17,944 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/B/c486d1ec51ad46579fc29e4a4cbe47b8 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/c486d1ec51ad46579fc29e4a4cbe47b8 2024-11-23T14:27:17,945 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/A/63e9792bae4341279862609258c1be04 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/63e9792bae4341279862609258c1be04 2024-11-23T14:27:17,949 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3d5e64ff6f2c8eec1637e7e45ace6102/B of 3d5e64ff6f2c8eec1637e7e45ace6102 into c486d1ec51ad46579fc29e4a4cbe47b8(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:27:17,949 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3d5e64ff6f2c8eec1637e7e45ace6102: 2024-11-23T14:27:17,949 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., storeName=3d5e64ff6f2c8eec1637e7e45ace6102/B, priority=13, startTime=1732372037507; duration=0sec 2024-11-23T14:27:17,949 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:27:17,949 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3d5e64ff6f2c8eec1637e7e45ace6102:B 2024-11-23T14:27:17,949 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:27:17,950 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3d5e64ff6f2c8eec1637e7e45ace6102/A of 3d5e64ff6f2c8eec1637e7e45ace6102 into 63e9792bae4341279862609258c1be04(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:27:17,950 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3d5e64ff6f2c8eec1637e7e45ace6102: 2024-11-23T14:27:17,950 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., storeName=3d5e64ff6f2c8eec1637e7e45ace6102/A, priority=13, startTime=1732372037507; duration=0sec 2024-11-23T14:27:17,950 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:27:17,950 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3d5e64ff6f2c8eec1637e7e45ace6102:A 2024-11-23T14:27:17,951 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36443 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:27:17,951 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1541): 3d5e64ff6f2c8eec1637e7e45ace6102/C is initiating minor compaction (all files) 2024-11-23T14:27:17,951 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3d5e64ff6f2c8eec1637e7e45ace6102/C in TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:17,951 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/58cf4f7bb37f449588942376ef735ccd, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/b552ac49af494df8ad9e95934a5a10f0, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/8dbea164d23e4c8caa6213550b12bd6f] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp, totalSize=35.6 K 2024-11-23T14:27:17,951 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 58cf4f7bb37f449588942376ef735ccd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1732372033881 2024-11-23T14:27:17,951 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting b552ac49af494df8ad9e95934a5a10f0, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1732372033906 2024-11-23T14:27:17,951 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 8dbea164d23e4c8caa6213550b12bd6f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1732372035117 2024-11-23T14:27:17,957 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3d5e64ff6f2c8eec1637e7e45ace6102#C#compaction#440 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:27:17,957 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/C/a3b3c332beb042889d19a073bd1dd8f2 is 50, key is test_row_0/C:col10/1732372035120/Put/seqid=0 2024-11-23T14:27:17,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742345_1521 (size=12493) 2024-11-23T14:27:18,056 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/A/24801b89b8a742adbaf0d5f240dcb832 2024-11-23T14:27:18,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/B/0955be0a156a4974861e7fa0a3eb63d2 is 50, key is test_row_0/B:col10/1732372036294/Put/seqid=0 2024-11-23T14:27:18,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742346_1522 (size=12151) 2024-11-23T14:27:18,067 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/B/0955be0a156a4974861e7fa0a3eb63d2 2024-11-23T14:27:18,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/C/ebe62d163e254180a2a8f00b6474a384 is 50, key is test_row_0/C:col10/1732372036294/Put/seqid=0 2024-11-23T14:27:18,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742347_1523 (size=12151) 2024-11-23T14:27:18,368 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/C/a3b3c332beb042889d19a073bd1dd8f2 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/a3b3c332beb042889d19a073bd1dd8f2 2024-11-23T14:27:18,371 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3d5e64ff6f2c8eec1637e7e45ace6102/C of 3d5e64ff6f2c8eec1637e7e45ace6102 into a3b3c332beb042889d19a073bd1dd8f2(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:27:18,371 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3d5e64ff6f2c8eec1637e7e45ace6102: 2024-11-23T14:27:18,371 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., storeName=3d5e64ff6f2c8eec1637e7e45ace6102/C, priority=13, startTime=1732372037507; duration=0sec 2024-11-23T14:27:18,371 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:27:18,371 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3d5e64ff6f2c8eec1637e7e45ace6102:C 2024-11-23T14:27:18,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on 3d5e64ff6f2c8eec1637e7e45ace6102 2024-11-23T14:27:18,454 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. as already flushing 2024-11-23T14:27:18,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:18,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32840 deadline: 1732372098477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:18,484 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:18,484 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:18,484 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:18,486 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/C/ebe62d163e254180a2a8f00b6474a384 2024-11-23T14:27:18,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:18,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32862 deadline: 1732372098479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:18,488 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:18,488 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:18,488 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:18,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/A/24801b89b8a742adbaf0d5f240dcb832 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/24801b89b8a742adbaf0d5f240dcb832 2024-11-23T14:27:18,489 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:18,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32854 deadline: 1732372098483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:18,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:18,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32790 deadline: 1732372098484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:18,490 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:18,490 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:18,490 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:18,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:18,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:18,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:18,493 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/24801b89b8a742adbaf0d5f240dcb832, entries=150, sequenceid=169, filesize=11.9 K 2024-11-23T14:27:18,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/B/0955be0a156a4974861e7fa0a3eb63d2 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/0955be0a156a4974861e7fa0a3eb63d2 2024-11-23T14:27:18,496 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/0955be0a156a4974861e7fa0a3eb63d2, entries=150, sequenceid=169, filesize=11.9 K 2024-11-23T14:27:18,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/C/ebe62d163e254180a2a8f00b6474a384 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/ebe62d163e254180a2a8f00b6474a384 2024-11-23T14:27:18,501 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/ebe62d163e254180a2a8f00b6474a384, entries=150, sequenceid=169, filesize=11.9 K 2024-11-23T14:27:18,501 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 3d5e64ff6f2c8eec1637e7e45ace6102 in 853ms, sequenceid=169, compaction requested=false 2024-11-23T14:27:18,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for 3d5e64ff6f2c8eec1637e7e45ace6102: 2024-11-23T14:27:18,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:18,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-23T14:27:18,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-23T14:27:18,504 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-23T14:27:18,504 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7690 sec 2024-11-23T14:27:18,506 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 1.7730 sec 2024-11-23T14:27:18,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on 3d5e64ff6f2c8eec1637e7e45ace6102 2024-11-23T14:27:18,592 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T14:27:18,592 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3d5e64ff6f2c8eec1637e7e45ace6102 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-23T14:27:18,597 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/A/fe3ae34564384756a95b94a414369998 is 50, key is test_row_0/A:col10/1732372038591/Put/seqid=0 2024-11-23T14:27:18,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742348_1524 (size=16931) 2024-11-23T14:27:18,600 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/A/fe3ae34564384756a95b94a414369998 2024-11-23T14:27:18,607 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/B/952d1fcdb09242f693c6c5342aabc19b is 50, key is test_row_0/B:col10/1732372038591/Put/seqid=0 2024-11-23T14:27:18,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742349_1525 (size=12151) 2024-11-23T14:27:18,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:18,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32790 deadline: 1732372098611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:18,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:18,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:18,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:18,619 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:18,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32854 deadline: 1732372098617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:18,619 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:18,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32862 deadline: 1732372098617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:18,620 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:18,620 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:18,620 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:18,621 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:18,621 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:18,621 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:18,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:18,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32840 deadline: 1732372098618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:18,627 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:18,627 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:18,627 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:18,734 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:18,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32790 deadline: 1732372098730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:18,734 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:18,734 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:18,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32862 deadline: 1732372098730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:18,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:18,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:18,735 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:18,735 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:18,735 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:18,736 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:18,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32854 deadline: 1732372098730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:18,736 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:18,736 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:18,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32840 deadline: 1732372098730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:18,736 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:18,736 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:18,736 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:18,736 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:18,736 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:18,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-23T14:27:18,880 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T14:27:18,881 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-23T14:27:18,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-11-23T14:27:18,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-23T14:27:18,882 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T14:27:18,883 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T14:27:18,883 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T14:27:18,942 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:18,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32790 deadline: 1732372098940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:18,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:18,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32862 deadline: 1732372098940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:18,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:18,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:18,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:18,943 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:18,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:18,943 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:18,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32854 deadline: 1732372098941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:18,943 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:18,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:18,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32840 deadline: 1732372098941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:18,943 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:18,943 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:18,943 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:18,943 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:18,943 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:18,943 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:18,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-23T14:27:19,010 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/B/952d1fcdb09242f693c6c5342aabc19b 2024-11-23T14:27:19,015 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/C/7db6442f16804dda8339807d9f5a92f2 is 50, key is test_row_0/C:col10/1732372038591/Put/seqid=0 2024-11-23T14:27:19,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742350_1526 (size=12151) 2024-11-23T14:27:19,035 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-23T14:27:19,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:19,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. as already flushing 2024-11-23T14:27:19,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:19,035 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:19,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:19,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:19,187 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-23T14:27:19,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:19,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. as already flushing 2024-11-23T14:27:19,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:19,188 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:19,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:19,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:19,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-23T14:27:19,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:19,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32790 deadline: 1732372099250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:19,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:19,252 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:19,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32862 deadline: 1732372099250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:19,252 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:19,253 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:19,253 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:19,253 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:19,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32854 deadline: 1732372099250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:19,253 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:19,253 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:19,253 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:19,253 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:19,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32840 deadline: 1732372099251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:19,253 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:19,253 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:19,253 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:19,253 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:19,253 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:19,340 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-23T14:27:19,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:19,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. as already flushing 2024-11-23T14:27:19,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:19,341 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:19,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:19,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:19,420 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/C/7db6442f16804dda8339807d9f5a92f2 2024-11-23T14:27:19,424 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/A/fe3ae34564384756a95b94a414369998 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/fe3ae34564384756a95b94a414369998 2024-11-23T14:27:19,427 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/fe3ae34564384756a95b94a414369998, entries=250, sequenceid=194, filesize=16.5 K 2024-11-23T14:27:19,428 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/B/952d1fcdb09242f693c6c5342aabc19b as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/952d1fcdb09242f693c6c5342aabc19b 2024-11-23T14:27:19,431 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/952d1fcdb09242f693c6c5342aabc19b, entries=150, sequenceid=194, filesize=11.9 K 2024-11-23T14:27:19,432 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/C/7db6442f16804dda8339807d9f5a92f2 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/7db6442f16804dda8339807d9f5a92f2 2024-11-23T14:27:19,435 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/7db6442f16804dda8339807d9f5a92f2, entries=150, sequenceid=194, filesize=11.9 K 2024-11-23T14:27:19,436 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 3d5e64ff6f2c8eec1637e7e45ace6102 in 844ms, sequenceid=194, compaction requested=true 2024-11-23T14:27:19,436 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3d5e64ff6f2c8eec1637e7e45ace6102: 2024-11-23T14:27:19,436 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:27:19,437 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41575 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:27:19,437 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1541): 3d5e64ff6f2c8eec1637e7e45ace6102/A is initiating minor compaction (all files) 2024-11-23T14:27:19,437 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3d5e64ff6f2c8eec1637e7e45ace6102/A in TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:19,437 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/63e9792bae4341279862609258c1be04, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/24801b89b8a742adbaf0d5f240dcb832, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/fe3ae34564384756a95b94a414369998] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp, totalSize=40.6 K 2024-11-23T14:27:19,437 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 63e9792bae4341279862609258c1be04, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1732372035117 2024-11-23T14:27:19,438 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24801b89b8a742adbaf0d5f240dcb832, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732372036294 2024-11-23T14:27:19,438 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe3ae34564384756a95b94a414369998, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732372038478 2024-11-23T14:27:19,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3d5e64ff6f2c8eec1637e7e45ace6102:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T14:27:19,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:27:19,443 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:27:19,443 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:27:19,443 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1541): 3d5e64ff6f2c8eec1637e7e45ace6102/B is initiating minor compaction (all files) 2024-11-23T14:27:19,443 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3d5e64ff6f2c8eec1637e7e45ace6102/B in TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:19,444 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/c486d1ec51ad46579fc29e4a4cbe47b8, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/0955be0a156a4974861e7fa0a3eb63d2, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/952d1fcdb09242f693c6c5342aabc19b] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp, totalSize=35.9 K 2024-11-23T14:27:19,444 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting c486d1ec51ad46579fc29e4a4cbe47b8, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1732372035117 2024-11-23T14:27:19,444 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 0955be0a156a4974861e7fa0a3eb63d2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732372036294 2024-11-23T14:27:19,444 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 952d1fcdb09242f693c6c5342aabc19b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732372038483 2024-11-23T14:27:19,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3d5e64ff6f2c8eec1637e7e45ace6102:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T14:27:19,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:27:19,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3d5e64ff6f2c8eec1637e7e45ace6102:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T14:27:19,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:27:19,450 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3d5e64ff6f2c8eec1637e7e45ace6102#A#compaction#446 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:27:19,450 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3d5e64ff6f2c8eec1637e7e45ace6102#B#compaction#447 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:27:19,451 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/B/29e7efb03cc045ea909e2125c7659438 is 50, key is test_row_0/B:col10/1732372038591/Put/seqid=0 2024-11-23T14:27:19,451 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/A/4c2b779d046048db98334ce7feca0557 is 50, key is test_row_0/A:col10/1732372038591/Put/seqid=0 2024-11-23T14:27:19,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742352_1528 (size=12595) 2024-11-23T14:27:19,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742351_1527 (size=12595) 2024-11-23T14:27:19,466 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/A/4c2b779d046048db98334ce7feca0557 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/4c2b779d046048db98334ce7feca0557 2024-11-23T14:27:19,472 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3d5e64ff6f2c8eec1637e7e45ace6102/A of 3d5e64ff6f2c8eec1637e7e45ace6102 into 4c2b779d046048db98334ce7feca0557(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:27:19,472 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3d5e64ff6f2c8eec1637e7e45ace6102: 2024-11-23T14:27:19,472 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., storeName=3d5e64ff6f2c8eec1637e7e45ace6102/A, priority=13, startTime=1732372039436; duration=0sec 2024-11-23T14:27:19,472 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:27:19,473 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3d5e64ff6f2c8eec1637e7e45ace6102:A 2024-11-23T14:27:19,473 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:27:19,475 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:27:19,475 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1541): 3d5e64ff6f2c8eec1637e7e45ace6102/C is initiating minor compaction (all files) 2024-11-23T14:27:19,475 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3d5e64ff6f2c8eec1637e7e45ace6102/C in TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:19,475 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/a3b3c332beb042889d19a073bd1dd8f2, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/ebe62d163e254180a2a8f00b6474a384, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/7db6442f16804dda8339807d9f5a92f2] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp, totalSize=35.9 K 2024-11-23T14:27:19,476 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting a3b3c332beb042889d19a073bd1dd8f2, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1732372035117 2024-11-23T14:27:19,476 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting ebe62d163e254180a2a8f00b6474a384, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732372036294 2024-11-23T14:27:19,476 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7db6442f16804dda8339807d9f5a92f2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732372038483 2024-11-23T14:27:19,483 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3d5e64ff6f2c8eec1637e7e45ace6102#C#compaction#448 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:27:19,483 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/C/e6ac4667ca624663b2fb01b84d429ba4 is 50, key is test_row_0/C:col10/1732372038591/Put/seqid=0 2024-11-23T14:27:19,493 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-23T14:27:19,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:19,493 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing 3d5e64ff6f2c8eec1637e7e45ace6102 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-23T14:27:19,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742353_1529 (size=12595) 2024-11-23T14:27:19,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/A/86154e038ddd4aeb8fb7dd60711884f7 is 50, key is test_row_0/A:col10/1732372038616/Put/seqid=0 2024-11-23T14:27:19,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742354_1530 (size=12151) 2024-11-23T14:27:19,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-23T14:27:19,763 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. as already flushing 2024-11-23T14:27:19,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on 3d5e64ff6f2c8eec1637e7e45ace6102 2024-11-23T14:27:19,829 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:19,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32862 deadline: 1732372099820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:19,829 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:19,829 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:19,829 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:19,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32790 deadline: 1732372099821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:19,829 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:19,830 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:19,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32854 deadline: 1732372099824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:19,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:19,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:19,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:19,830 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:19,830 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:19,830 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:19,836 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:19,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32840 deadline: 1732372099830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:19,836 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:19,836 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:19,836 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:19,863 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/B/29e7efb03cc045ea909e2125c7659438 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/29e7efb03cc045ea909e2125c7659438 2024-11-23T14:27:19,867 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3d5e64ff6f2c8eec1637e7e45ace6102/B of 3d5e64ff6f2c8eec1637e7e45ace6102 into 29e7efb03cc045ea909e2125c7659438(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:27:19,867 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3d5e64ff6f2c8eec1637e7e45ace6102: 2024-11-23T14:27:19,867 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., storeName=3d5e64ff6f2c8eec1637e7e45ace6102/B, priority=13, startTime=1732372039442; duration=0sec 2024-11-23T14:27:19,867 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:27:19,867 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3d5e64ff6f2c8eec1637e7e45ace6102:B 2024-11-23T14:27:19,898 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/C/e6ac4667ca624663b2fb01b84d429ba4 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/e6ac4667ca624663b2fb01b84d429ba4 2024-11-23T14:27:19,901 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/A/86154e038ddd4aeb8fb7dd60711884f7 2024-11-23T14:27:19,903 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3d5e64ff6f2c8eec1637e7e45ace6102/C of 3d5e64ff6f2c8eec1637e7e45ace6102 into e6ac4667ca624663b2fb01b84d429ba4(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:27:19,903 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3d5e64ff6f2c8eec1637e7e45ace6102: 2024-11-23T14:27:19,903 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., storeName=3d5e64ff6f2c8eec1637e7e45ace6102/C, priority=13, startTime=1732372039448; duration=0sec 2024-11-23T14:27:19,903 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:27:19,903 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3d5e64ff6f2c8eec1637e7e45ace6102:C 2024-11-23T14:27:19,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/B/904f1b4f71bf4b3394ac1e1befc43a6d is 50, key is test_row_0/B:col10/1732372038616/Put/seqid=0 2024-11-23T14:27:19,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742355_1531 (size=12151) 2024-11-23T14:27:19,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:19,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32862 deadline: 1732372099940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:19,942 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:19,942 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:19,942 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:19,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32840 deadline: 1732372099941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:19,942 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:19,942 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:19,942 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:19,942 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:19,942 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:19,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32790 deadline: 1732372099941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:19,942 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:19,942 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:19,942 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:19,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:19,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32854 deadline: 1732372099941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:19,949 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:19,950 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:19,950 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:20,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-23T14:27:20,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:20,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32862 deadline: 1732372100150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:20,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:20,155 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:20,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32840 deadline: 1732372100150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:20,155 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:20,155 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:20,155 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:20,155 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:20,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:20,155 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:20,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32790 deadline: 1732372100150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:20,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:20,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:20,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:20,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:20,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32854 deadline: 1732372100160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:20,163 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:20,163 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:20,164 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:20,313 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/B/904f1b4f71bf4b3394ac1e1befc43a6d 2024-11-23T14:27:20,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/C/bdb58ddf4477436596dcade7c22e96e2 is 50, key is test_row_0/C:col10/1732372038616/Put/seqid=0 2024-11-23T14:27:20,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742356_1532 (size=12151) 2024-11-23T14:27:20,465 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:20,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32862 deadline: 1732372100460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:20,465 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:20,465 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:20,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32840 deadline: 1732372100460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:20,465 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:20,465 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:20,465 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:20,465 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:20,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:20,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32790 deadline: 1732372100460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:20,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:20,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:20,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:20,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:20,474 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:20,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32854 deadline: 1732372100470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:20,475 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:20,475 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:20,475 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:20,726 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/C/bdb58ddf4477436596dcade7c22e96e2 2024-11-23T14:27:20,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/A/86154e038ddd4aeb8fb7dd60711884f7 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/86154e038ddd4aeb8fb7dd60711884f7 2024-11-23T14:27:20,732 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/86154e038ddd4aeb8fb7dd60711884f7, entries=150, sequenceid=209, filesize=11.9 K 2024-11-23T14:27:20,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/B/904f1b4f71bf4b3394ac1e1befc43a6d as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/904f1b4f71bf4b3394ac1e1befc43a6d 2024-11-23T14:27:20,737 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/904f1b4f71bf4b3394ac1e1befc43a6d, entries=150, sequenceid=209, filesize=11.9 K 2024-11-23T14:27:20,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/C/bdb58ddf4477436596dcade7c22e96e2 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/bdb58ddf4477436596dcade7c22e96e2 2024-11-23T14:27:20,741 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/bdb58ddf4477436596dcade7c22e96e2, entries=150, sequenceid=209, filesize=11.9 K 2024-11-23T14:27:20,741 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 3d5e64ff6f2c8eec1637e7e45ace6102 in 1248ms, sequenceid=209, compaction requested=false 2024-11-23T14:27:20,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for 3d5e64ff6f2c8eec1637e7e45ace6102: 2024-11-23T14:27:20,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:20,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-11-23T14:27:20,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-11-23T14:27:20,744 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-23T14:27:20,744 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8590 sec 2024-11-23T14:27:20,745 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 1.8630 sec 2024-11-23T14:27:20,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on 3d5e64ff6f2c8eec1637e7e45ace6102 2024-11-23T14:27:20,972 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T14:27:20,972 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3d5e64ff6f2c8eec1637e7e45ace6102 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-23T14:27:20,976 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/A/f72b08d40b9f48268d1641eeedcf5d11 is 50, key is test_row_0/A:col10/1732372040971/Put/seqid=0 2024-11-23T14:27:20,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742357_1533 (size=14541) 2024-11-23T14:27:20,979 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/A/f72b08d40b9f48268d1641eeedcf5d11 2024-11-23T14:27:20,986 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/B/f3dfb676b57f41b19915a74662fc3e75 is 50, key is test_row_0/B:col10/1732372040971/Put/seqid=0 2024-11-23T14:27:20,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742358_1534 (size=12151) 2024-11-23T14:27:21,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:21,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32862 deadline: 1732372101002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:21,005 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:21,006 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:21,006 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:21,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:21,012 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:21,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32790 deadline: 1732372101003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:21,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32854 deadline: 1732372101004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:21,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:21,012 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:21,012 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:21,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32840 deadline: 1732372101005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:21,012 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:21,012 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:21,012 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:21,012 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:21,012 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:21,012 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:21,012 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:21,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-23T14:27:21,030 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T14:27:21,031 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-23T14:27:21,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees 2024-11-23T14:27:21,032 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T14:27:21,033 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T14:27:21,033 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T14:27:21,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-23T14:27:21,112 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:21,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32862 deadline: 1732372101110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:21,113 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:21,113 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:21,113 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:21,125 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:21,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32790 deadline: 1732372101120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:21,126 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:21,126 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:21,126 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:21,126 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:21,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32854 deadline: 1732372101120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:21,126 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:21,126 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:21,126 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:21,126 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:21,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32840 deadline: 1732372101120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:21,126 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:21,126 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:21,126 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:21,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-23T14:27:21,184 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-23T14:27:21,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:21,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. as already flushing 2024-11-23T14:27:21,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:21,185 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:21,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:21,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:21,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:21,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32862 deadline: 1732372101320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:21,320 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:21,320 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:21,320 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:21,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:21,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32790 deadline: 1732372101330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:21,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:21,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:21,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:21,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:21,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32854 deadline: 1732372101330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:21,336 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:21,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:21,336 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:21,336 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:21,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32840 deadline: 1732372101330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:21,337 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:21,337 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:21,337 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:21,337 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-23T14:27:21,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:21,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. as already flushing 2024-11-23T14:27:21,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:21,337 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:21,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:21,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:21,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-23T14:27:21,396 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/B/f3dfb676b57f41b19915a74662fc3e75 2024-11-23T14:27:21,402 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/C/3b1f36063c384c4c972f5f894177fbea is 50, key is test_row_0/C:col10/1732372040971/Put/seqid=0 2024-11-23T14:27:21,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742359_1535 (size=12151) 2024-11-23T14:27:21,490 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-23T14:27:21,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:21,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. as already flushing 2024-11-23T14:27:21,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:21,490 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:21,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:21,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:21,637 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:21,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32862 deadline: 1732372101630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:21,637 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:21,637 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:21,638 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:21,642 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-23T14:27:21,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:21,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. as already flushing 2024-11-23T14:27:21,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:21,643 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:21,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:21,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:21,646 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:21,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32790 deadline: 1732372101640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:21,647 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:21,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32854 deadline: 1732372101640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:21,647 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:21,647 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:21,647 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:21,647 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:21,647 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:21,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32840 deadline: 1732372101640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:21,647 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:21,647 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:21,647 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:21,648 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:21,648 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:21,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-23T14:27:21,795 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-23T14:27:21,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:21,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. as already flushing 2024-11-23T14:27:21,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:21,796 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:21,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:21,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:21,805 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/C/3b1f36063c384c4c972f5f894177fbea 2024-11-23T14:27:21,809 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/A/f72b08d40b9f48268d1641eeedcf5d11 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/f72b08d40b9f48268d1641eeedcf5d11 2024-11-23T14:27:21,812 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/f72b08d40b9f48268d1641eeedcf5d11, entries=200, sequenceid=234, filesize=14.2 K 2024-11-23T14:27:21,812 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/B/f3dfb676b57f41b19915a74662fc3e75 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/f3dfb676b57f41b19915a74662fc3e75 2024-11-23T14:27:21,817 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/f3dfb676b57f41b19915a74662fc3e75, entries=150, sequenceid=234, filesize=11.9 K 2024-11-23T14:27:21,818 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/C/3b1f36063c384c4c972f5f894177fbea as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/3b1f36063c384c4c972f5f894177fbea 2024-11-23T14:27:21,820 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/3b1f36063c384c4c972f5f894177fbea, entries=150, sequenceid=234, filesize=11.9 K 2024-11-23T14:27:21,821 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 3d5e64ff6f2c8eec1637e7e45ace6102 in 848ms, sequenceid=234, compaction requested=true 2024-11-23T14:27:21,821 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3d5e64ff6f2c8eec1637e7e45ace6102: 2024-11-23T14:27:21,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3d5e64ff6f2c8eec1637e7e45ace6102:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T14:27:21,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:27:21,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3d5e64ff6f2c8eec1637e7e45ace6102:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T14:27:21,821 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:27:21,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:27:21,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3d5e64ff6f2c8eec1637e7e45ace6102:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T14:27:21,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:27:21,821 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:27:21,821 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39287 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:27:21,821 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:27:21,821 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1541): 3d5e64ff6f2c8eec1637e7e45ace6102/B is initiating minor compaction (all files) 2024-11-23T14:27:21,821 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1541): 3d5e64ff6f2c8eec1637e7e45ace6102/A is initiating minor compaction (all files) 2024-11-23T14:27:21,822 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3d5e64ff6f2c8eec1637e7e45ace6102/A in TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:21,822 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3d5e64ff6f2c8eec1637e7e45ace6102/B in TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:21,822 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/4c2b779d046048db98334ce7feca0557, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/86154e038ddd4aeb8fb7dd60711884f7, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/f72b08d40b9f48268d1641eeedcf5d11] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp, totalSize=38.4 K 2024-11-23T14:27:21,822 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/29e7efb03cc045ea909e2125c7659438, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/904f1b4f71bf4b3394ac1e1befc43a6d, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/f3dfb676b57f41b19915a74662fc3e75] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp, totalSize=36.0 K 2024-11-23T14:27:21,822 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c2b779d046048db98334ce7feca0557, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732372038483 2024-11-23T14:27:21,822 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 29e7efb03cc045ea909e2125c7659438, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732372038483 2024-11-23T14:27:21,822 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 86154e038ddd4aeb8fb7dd60711884f7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1732372038615 2024-11-23T14:27:21,822 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 904f1b4f71bf4b3394ac1e1befc43a6d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1732372038615 2024-11-23T14:27:21,825 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting f3dfb676b57f41b19915a74662fc3e75, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1732372039819 2024-11-23T14:27:21,825 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting f72b08d40b9f48268d1641eeedcf5d11, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1732372039819 2024-11-23T14:27:21,829 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3d5e64ff6f2c8eec1637e7e45ace6102#A#compaction#455 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:27:21,830 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/A/94debe76664643cd9d094eda878b5d2b is 50, key is test_row_0/A:col10/1732372040971/Put/seqid=0 2024-11-23T14:27:21,830 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3d5e64ff6f2c8eec1637e7e45ace6102#B#compaction#456 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:27:21,830 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/B/fbf0fdd76e6d485799ca8eceeb9c3bf9 is 50, key is test_row_0/B:col10/1732372040971/Put/seqid=0 2024-11-23T14:27:21,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742360_1536 (size=12697) 2024-11-23T14:27:21,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742361_1537 (size=12697) 2024-11-23T14:27:21,948 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-23T14:27:21,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:21,948 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2902): Flushing 3d5e64ff6f2c8eec1637e7e45ace6102 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-23T14:27:21,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/A/1930e6d09ca84f0da81540256f573090 is 50, key is test_row_0/A:col10/1732372041002/Put/seqid=0 2024-11-23T14:27:21,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742362_1538 (size=12151) 2024-11-23T14:27:22,153 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. as already flushing 2024-11-23T14:27:22,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on 3d5e64ff6f2c8eec1637e7e45ace6102 2024-11-23T14:27:22,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-23T14:27:22,211 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:22,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32854 deadline: 1732372102207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:22,211 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:22,211 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:22,211 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:22,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:22,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32862 deadline: 1732372102209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:22,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:22,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32790 deadline: 1732372102210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:22,220 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:22,220 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:22,220 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:22,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:22,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:22,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:22,222 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:22,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32840 deadline: 1732372102211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:22,222 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:22,222 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:22,222 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:22,237 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/B/fbf0fdd76e6d485799ca8eceeb9c3bf9 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/fbf0fdd76e6d485799ca8eceeb9c3bf9 2024-11-23T14:27:22,240 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/A/94debe76664643cd9d094eda878b5d2b as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/94debe76664643cd9d094eda878b5d2b 2024-11-23T14:27:22,241 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3d5e64ff6f2c8eec1637e7e45ace6102/B of 3d5e64ff6f2c8eec1637e7e45ace6102 into fbf0fdd76e6d485799ca8eceeb9c3bf9(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:27:22,241 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3d5e64ff6f2c8eec1637e7e45ace6102: 2024-11-23T14:27:22,241 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., storeName=3d5e64ff6f2c8eec1637e7e45ace6102/B, priority=13, startTime=1732372041821; duration=0sec 2024-11-23T14:27:22,241 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:27:22,241 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3d5e64ff6f2c8eec1637e7e45ace6102:B 2024-11-23T14:27:22,241 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:27:22,241 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:27:22,241 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1541): 3d5e64ff6f2c8eec1637e7e45ace6102/C is initiating minor compaction (all files) 2024-11-23T14:27:22,241 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3d5e64ff6f2c8eec1637e7e45ace6102/C in TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:22,242 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/e6ac4667ca624663b2fb01b84d429ba4, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/bdb58ddf4477436596dcade7c22e96e2, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/3b1f36063c384c4c972f5f894177fbea] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp, totalSize=36.0 K 2024-11-23T14:27:22,242 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting e6ac4667ca624663b2fb01b84d429ba4, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732372038483 2024-11-23T14:27:22,242 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting bdb58ddf4477436596dcade7c22e96e2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1732372038615 2024-11-23T14:27:22,242 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b1f36063c384c4c972f5f894177fbea, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1732372039819 2024-11-23T14:27:22,243 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3d5e64ff6f2c8eec1637e7e45ace6102/A of 3d5e64ff6f2c8eec1637e7e45ace6102 into 94debe76664643cd9d094eda878b5d2b(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:27:22,244 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3d5e64ff6f2c8eec1637e7e45ace6102: 2024-11-23T14:27:22,244 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., storeName=3d5e64ff6f2c8eec1637e7e45ace6102/A, priority=13, startTime=1732372041821; duration=0sec 2024-11-23T14:27:22,244 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:27:22,244 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3d5e64ff6f2c8eec1637e7e45ace6102:A 2024-11-23T14:27:22,246 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3d5e64ff6f2c8eec1637e7e45ace6102#C#compaction#458 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:27:22,247 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/C/b36a7b9c0c954713a2e39445deb63b8b is 50, key is test_row_0/C:col10/1732372040971/Put/seqid=0 2024-11-23T14:27:22,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742363_1539 (size=12697) 2024-11-23T14:27:22,255 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/C/b36a7b9c0c954713a2e39445deb63b8b as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/b36a7b9c0c954713a2e39445deb63b8b 2024-11-23T14:27:22,259 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3d5e64ff6f2c8eec1637e7e45ace6102/C of 3d5e64ff6f2c8eec1637e7e45ace6102 into b36a7b9c0c954713a2e39445deb63b8b(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:27:22,259 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3d5e64ff6f2c8eec1637e7e45ace6102: 2024-11-23T14:27:22,259 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., storeName=3d5e64ff6f2c8eec1637e7e45ace6102/C, priority=13, startTime=1732372041821; duration=0sec 2024-11-23T14:27:22,260 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:27:22,260 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3d5e64ff6f2c8eec1637e7e45ace6102:C 2024-11-23T14:27:22,324 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:22,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32854 deadline: 1732372102320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:22,325 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:22,325 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:22,325 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:22,334 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:22,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32862 deadline: 1732372102330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:22,334 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:22,334 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:22,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32790 deadline: 1732372102330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:22,334 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:22,334 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:22,335 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:22,334 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:22,335 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:22,335 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:22,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32840 deadline: 1732372102330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:22,335 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:22,335 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:22,335 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:22,355 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/A/1930e6d09ca84f0da81540256f573090 2024-11-23T14:27:22,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/B/70cd4709c55d485b9d7f246f586a7206 is 50, key is test_row_0/B:col10/1732372041002/Put/seqid=0 2024-11-23T14:27:22,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742364_1540 (size=12151) 2024-11-23T14:27:22,534 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:22,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32854 deadline: 1732372102530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:22,534 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:22,534 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:22,534 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:22,543 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:22,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32862 deadline: 1732372102540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:22,544 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:22,544 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:22,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32790 deadline: 1732372102540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:22,544 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:22,544 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:22,544 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:22,544 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:22,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32840 deadline: 1732372102540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:22,544 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:22,544 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:22,544 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:22,544 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:22,544 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:22,764 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/B/70cd4709c55d485b9d7f246f586a7206 2024-11-23T14:27:22,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/C/ccd8920fbf5d47bc8a346e139b3c75e2 is 50, key is test_row_0/C:col10/1732372041002/Put/seqid=0 2024-11-23T14:27:22,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742365_1541 (size=12151) 2024-11-23T14:27:22,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:22,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32854 deadline: 1732372102840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:22,846 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:22,846 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:22,846 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:22,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:22,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32862 deadline: 1732372102850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:22,856 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:22,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:22,856 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:22,856 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:22,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32790 deadline: 1732372102850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:22,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:22,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:22,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:22,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:22,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32840 deadline: 1732372102850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:22,857 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:22,857 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:22,857 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:23,173 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/C/ccd8920fbf5d47bc8a346e139b3c75e2 2024-11-23T14:27:23,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/A/1930e6d09ca84f0da81540256f573090 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/1930e6d09ca84f0da81540256f573090 2024-11-23T14:27:23,179 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/1930e6d09ca84f0da81540256f573090, entries=150, sequenceid=247, filesize=11.9 K 2024-11-23T14:27:23,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/B/70cd4709c55d485b9d7f246f586a7206 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/70cd4709c55d485b9d7f246f586a7206 2024-11-23T14:27:23,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-23T14:27:23,182 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/70cd4709c55d485b9d7f246f586a7206, entries=150, sequenceid=247, filesize=11.9 K 2024-11-23T14:27:23,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/C/ccd8920fbf5d47bc8a346e139b3c75e2 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/ccd8920fbf5d47bc8a346e139b3c75e2 2024-11-23T14:27:23,186 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/ccd8920fbf5d47bc8a346e139b3c75e2, entries=150, sequenceid=247, filesize=11.9 K 2024-11-23T14:27:23,186 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 3d5e64ff6f2c8eec1637e7e45ace6102 in 1238ms, sequenceid=247, compaction requested=false 2024-11-23T14:27:23,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2603): Flush status journal for 3d5e64ff6f2c8eec1637e7e45ace6102: 2024-11-23T14:27:23,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:23,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=138 2024-11-23T14:27:23,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster(4169): Remote procedure done, pid=138 2024-11-23T14:27:23,189 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-23T14:27:23,189 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1550 sec 2024-11-23T14:27:23,190 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees in 2.1590 sec 2024-11-23T14:27:23,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on 3d5e64ff6f2c8eec1637e7e45ace6102 2024-11-23T14:27:23,364 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T14:27:23,364 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3d5e64ff6f2c8eec1637e7e45ace6102 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-23T14:27:23,368 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/A/7c364d794aa44313985efd9a49f6cd65 is 50, key is test_row_0/A:col10/1732372042210/Put/seqid=0 2024-11-23T14:27:23,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742366_1542 (size=17181) 2024-11-23T14:27:23,371 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/A/7c364d794aa44313985efd9a49f6cd65 2024-11-23T14:27:23,377 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/B/19f0b68ada014d11a2f9da277017385a is 50, key is test_row_0/B:col10/1732372042210/Put/seqid=0 2024-11-23T14:27:23,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742367_1543 (size=12301) 2024-11-23T14:27:23,383 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:23,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32790 deadline: 1732372103375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:23,384 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:23,384 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:23,384 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:23,385 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:23,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32862 deadline: 1732372103382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:23,386 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:23,385 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:23,386 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:23,386 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:23,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32854 deadline: 1732372103383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:23,386 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:23,386 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:23,386 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:23,391 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:23,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32840 deadline: 1732372103383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:23,391 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:23,391 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:23,391 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:23,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:23,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32790 deadline: 1732372103490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:23,494 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:23,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:23,494 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:23,494 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:23,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32862 deadline: 1732372103490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:23,494 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:23,494 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:23,494 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:23,494 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:23,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32854 deadline: 1732372103490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:23,494 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:23,494 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:23,494 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:23,503 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:23,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32840 deadline: 1732372103500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:23,503 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:23,503 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:23,503 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:23,704 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:23,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32790 deadline: 1732372103700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:23,705 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:23,705 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:23,705 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:23,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32862 deadline: 1732372103700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:23,705 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:23,705 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:23,705 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:23,705 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:23,705 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:23,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32854 deadline: 1732372103700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:23,705 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:23,705 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:23,705 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:23,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:23,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32840 deadline: 1732372103710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:23,714 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:23,714 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:23,714 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:23,781 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/B/19f0b68ada014d11a2f9da277017385a 2024-11-23T14:27:23,787 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/C/57e9ac60d9824733aa9341bf3c73c8a0 is 50, key is test_row_0/C:col10/1732372042210/Put/seqid=0 2024-11-23T14:27:23,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742368_1544 (size=12301) 2024-11-23T14:27:24,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:24,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32790 deadline: 1732372104010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:24,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:24,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:24,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:24,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:24,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32862 deadline: 1732372104010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:24,014 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:24,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:24,014 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:24,014 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:24,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32854 deadline: 1732372104010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:24,015 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:24,015 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:24,015 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:24,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:24,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32840 deadline: 1732372104020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:24,024 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:24,024 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:24,024 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:24,190 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/C/57e9ac60d9824733aa9341bf3c73c8a0 2024-11-23T14:27:24,192 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/A/7c364d794aa44313985efd9a49f6cd65 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/7c364d794aa44313985efd9a49f6cd65 2024-11-23T14:27:24,195 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/7c364d794aa44313985efd9a49f6cd65, entries=250, sequenceid=276, filesize=16.8 K 2024-11-23T14:27:24,196 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/B/19f0b68ada014d11a2f9da277017385a as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/19f0b68ada014d11a2f9da277017385a 2024-11-23T14:27:24,198 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/19f0b68ada014d11a2f9da277017385a, entries=150, sequenceid=276, filesize=12.0 K 2024-11-23T14:27:24,199 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/C/57e9ac60d9824733aa9341bf3c73c8a0 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/57e9ac60d9824733aa9341bf3c73c8a0 2024-11-23T14:27:24,201 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/57e9ac60d9824733aa9341bf3c73c8a0, entries=150, sequenceid=276, filesize=12.0 K 2024-11-23T14:27:24,202 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 3d5e64ff6f2c8eec1637e7e45ace6102 in 838ms, sequenceid=276, compaction requested=true 2024-11-23T14:27:24,202 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3d5e64ff6f2c8eec1637e7e45ace6102: 2024-11-23T14:27:24,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3d5e64ff6f2c8eec1637e7e45ace6102:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T14:27:24,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:27:24,202 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:27:24,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3d5e64ff6f2c8eec1637e7e45ace6102:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T14:27:24,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:27:24,202 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:27:24,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3d5e64ff6f2c8eec1637e7e45ace6102:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T14:27:24,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:27:24,203 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:27:24,203 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42029 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:27:24,203 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1541): 3d5e64ff6f2c8eec1637e7e45ace6102/A is initiating minor compaction (all files) 2024-11-23T14:27:24,203 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1541): 3d5e64ff6f2c8eec1637e7e45ace6102/B is initiating minor compaction (all files) 2024-11-23T14:27:24,203 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3d5e64ff6f2c8eec1637e7e45ace6102/B in TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:24,203 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/fbf0fdd76e6d485799ca8eceeb9c3bf9, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/70cd4709c55d485b9d7f246f586a7206, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/19f0b68ada014d11a2f9da277017385a] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp, totalSize=36.3 K 2024-11-23T14:27:24,203 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3d5e64ff6f2c8eec1637e7e45ace6102/A in TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:24,203 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/94debe76664643cd9d094eda878b5d2b, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/1930e6d09ca84f0da81540256f573090, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/7c364d794aa44313985efd9a49f6cd65] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp, totalSize=41.0 K 2024-11-23T14:27:24,203 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting fbf0fdd76e6d485799ca8eceeb9c3bf9, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1732372039819 2024-11-23T14:27:24,204 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 94debe76664643cd9d094eda878b5d2b, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1732372039819 2024-11-23T14:27:24,204 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 70cd4709c55d485b9d7f246f586a7206, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1732372041001 2024-11-23T14:27:24,204 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1930e6d09ca84f0da81540256f573090, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1732372041001 2024-11-23T14:27:24,204 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 19f0b68ada014d11a2f9da277017385a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1732372042210 2024-11-23T14:27:24,204 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c364d794aa44313985efd9a49f6cd65, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1732372042200 2024-11-23T14:27:24,209 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3d5e64ff6f2c8eec1637e7e45ace6102#B#compaction#464 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:27:24,209 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/B/ea1adfc7e6d74baa8d8a4998bb182be2 is 50, key is test_row_0/B:col10/1732372042210/Put/seqid=0 2024-11-23T14:27:24,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742369_1545 (size=12949) 2024-11-23T14:27:24,222 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3d5e64ff6f2c8eec1637e7e45ace6102#A#compaction#465 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:27:24,222 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/A/9a9b4454ed0148169e49e39e2a7b2c8b is 50, key is test_row_0/A:col10/1732372042210/Put/seqid=0 2024-11-23T14:27:24,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742370_1546 (size=12949) 2024-11-23T14:27:24,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on 3d5e64ff6f2c8eec1637e7e45ace6102 2024-11-23T14:27:24,525 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T14:27:24,525 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3d5e64ff6f2c8eec1637e7e45ace6102 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-23T14:27:24,528 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/A/05f3fddd57be4890a9e51984caf38eee is 50, key is test_row_0/A:col10/1732372044524/Put/seqid=0 2024-11-23T14:27:24,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742371_1547 (size=14741) 2024-11-23T14:27:24,566 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:24,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32790 deadline: 1732372104558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:24,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:24,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:24,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:24,569 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:24,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32854 deadline: 1732372104558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:24,569 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:24,569 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:24,569 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:24,569 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:24,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32862 deadline: 1732372104559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:24,569 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:24,569 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:24,569 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:24,569 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:24,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32840 deadline: 1732372104566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:24,570 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:24,570 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:24,570 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:24,623 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/B/ea1adfc7e6d74baa8d8a4998bb182be2 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/ea1adfc7e6d74baa8d8a4998bb182be2 2024-11-23T14:27:24,626 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3d5e64ff6f2c8eec1637e7e45ace6102/B of 3d5e64ff6f2c8eec1637e7e45ace6102 into ea1adfc7e6d74baa8d8a4998bb182be2(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:27:24,627 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3d5e64ff6f2c8eec1637e7e45ace6102: 2024-11-23T14:27:24,627 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., storeName=3d5e64ff6f2c8eec1637e7e45ace6102/B, priority=13, startTime=1732372044202; duration=0sec 2024-11-23T14:27:24,627 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:27:24,627 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3d5e64ff6f2c8eec1637e7e45ace6102:B 2024-11-23T14:27:24,627 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:27:24,627 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:27:24,627 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1541): 3d5e64ff6f2c8eec1637e7e45ace6102/C is initiating minor compaction (all files) 2024-11-23T14:27:24,627 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3d5e64ff6f2c8eec1637e7e45ace6102/C in TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:24,628 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/b36a7b9c0c954713a2e39445deb63b8b, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/ccd8920fbf5d47bc8a346e139b3c75e2, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/57e9ac60d9824733aa9341bf3c73c8a0] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp, totalSize=36.3 K 2024-11-23T14:27:24,628 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting b36a7b9c0c954713a2e39445deb63b8b, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1732372039819 2024-11-23T14:27:24,628 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting ccd8920fbf5d47bc8a346e139b3c75e2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1732372041001 2024-11-23T14:27:24,628 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 57e9ac60d9824733aa9341bf3c73c8a0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1732372042210 2024-11-23T14:27:24,632 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3d5e64ff6f2c8eec1637e7e45ace6102#C#compaction#467 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:27:24,633 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/C/dedd24e335774be5a3ca6ab2398383ba is 50, key is test_row_0/C:col10/1732372042210/Put/seqid=0 2024-11-23T14:27:24,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742372_1548 (size=12949) 2024-11-23T14:27:24,637 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/A/9a9b4454ed0148169e49e39e2a7b2c8b as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/9a9b4454ed0148169e49e39e2a7b2c8b 2024-11-23T14:27:24,639 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3d5e64ff6f2c8eec1637e7e45ace6102/A of 3d5e64ff6f2c8eec1637e7e45ace6102 into 9a9b4454ed0148169e49e39e2a7b2c8b(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:27:24,639 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3d5e64ff6f2c8eec1637e7e45ace6102: 2024-11-23T14:27:24,639 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., storeName=3d5e64ff6f2c8eec1637e7e45ace6102/A, priority=13, startTime=1732372044202; duration=0sec 2024-11-23T14:27:24,639 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:27:24,639 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3d5e64ff6f2c8eec1637e7e45ace6102:A 2024-11-23T14:27:24,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:24,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32790 deadline: 1732372104670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:24,676 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:24,676 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:24,676 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:24,683 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:24,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32854 deadline: 1732372104680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:24,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:24,684 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:24,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32862 deadline: 1732372104680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:24,684 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:24,684 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:24,684 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:24,684 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:24,684 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:24,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:24,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32840 deadline: 1732372104680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:24,684 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:24,684 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:24,684 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:24,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:24,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32790 deadline: 1732372104880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:24,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:24,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:24,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:24,892 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:24,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32854 deadline: 1732372104890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:24,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:24,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32862 deadline: 1732372104890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:24,893 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:24,893 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:24,893 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:24,893 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:24,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:24,893 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:24,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32840 deadline: 1732372104890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:24,893 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:24,893 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:24,893 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:24,893 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:24,931 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/A/05f3fddd57be4890a9e51984caf38eee 2024-11-23T14:27:24,936 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/B/ebba26bc23b44225aad1cfbd024e8957 is 50, key is test_row_0/B:col10/1732372044524/Put/seqid=0 2024-11-23T14:27:24,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742373_1549 (size=12301) 2024-11-23T14:27:25,039 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/C/dedd24e335774be5a3ca6ab2398383ba as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/dedd24e335774be5a3ca6ab2398383ba 2024-11-23T14:27:25,042 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3d5e64ff6f2c8eec1637e7e45ace6102/C of 3d5e64ff6f2c8eec1637e7e45ace6102 into dedd24e335774be5a3ca6ab2398383ba(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:27:25,043 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3d5e64ff6f2c8eec1637e7e45ace6102: 2024-11-23T14:27:25,043 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., storeName=3d5e64ff6f2c8eec1637e7e45ace6102/C, priority=13, startTime=1732372044202; duration=0sec 2024-11-23T14:27:25,043 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:27:25,043 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3d5e64ff6f2c8eec1637e7e45ace6102:C 2024-11-23T14:27:25,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-23T14:27:25,190 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T14:27:25,191 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-23T14:27:25,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees 2024-11-23T14:27:25,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-23T14:27:25,192 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T14:27:25,193 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T14:27:25,193 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T14:27:25,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:25,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32790 deadline: 1732372105190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:25,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:25,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:25,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:25,202 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:25,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32854 deadline: 1732372105199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:25,202 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:25,202 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:25,203 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:25,203 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:25,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32862 deadline: 1732372105200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:25,203 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:25,203 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:25,203 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:25,203 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:25,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32840 deadline: 1732372105200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:25,203 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:25,203 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:25,203 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:25,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-23T14:27:25,340 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/B/ebba26bc23b44225aad1cfbd024e8957 2024-11-23T14:27:25,345 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-23T14:27:25,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:25,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. as already flushing 2024-11-23T14:27:25,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:25,345 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:25,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:25,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:25,346 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/C/7dd5e0b989cc42a884e5f843fe93dbe1 is 50, key is test_row_0/C:col10/1732372044524/Put/seqid=0 2024-11-23T14:27:25,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742374_1550 (size=12301) 2024-11-23T14:27:25,497 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-23T14:27:25,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:25,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. as already flushing 2024-11-23T14:27:25,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:25,498 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:25,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:25,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:25,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-23T14:27:25,650 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-23T14:27:25,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:25,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. as already flushing 2024-11-23T14:27:25,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:25,650 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:25,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:25,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:25,714 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:25,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32790 deadline: 1732372105710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:25,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:25,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:25,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:25,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32854 deadline: 1732372105710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:25,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:25,715 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:25,715 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:25,715 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:25,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:25,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32862 deadline: 1732372105710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:25,715 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:25,715 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:25,715 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:25,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:25,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32840 deadline: 1732372105710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:25,716 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:25,716 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:25,716 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:25,750 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/C/7dd5e0b989cc42a884e5f843fe93dbe1 2024-11-23T14:27:25,802 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-23T14:27:25,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:25,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. as already flushing 2024-11-23T14:27:25,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:25,803 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:25,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:25,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:25,804 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/A/05f3fddd57be4890a9e51984caf38eee as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/05f3fddd57be4890a9e51984caf38eee 2024-11-23T14:27:25,807 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/05f3fddd57be4890a9e51984caf38eee, entries=200, sequenceid=287, filesize=14.4 K 2024-11-23T14:27:25,807 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/B/ebba26bc23b44225aad1cfbd024e8957 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/ebba26bc23b44225aad1cfbd024e8957 2024-11-23T14:27:25,810 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/ebba26bc23b44225aad1cfbd024e8957, entries=150, sequenceid=287, filesize=12.0 K 2024-11-23T14:27:25,811 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/C/7dd5e0b989cc42a884e5f843fe93dbe1 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/7dd5e0b989cc42a884e5f843fe93dbe1 2024-11-23T14:27:25,814 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/7dd5e0b989cc42a884e5f843fe93dbe1, entries=150, sequenceid=287, filesize=12.0 K 2024-11-23T14:27:25,815 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 3d5e64ff6f2c8eec1637e7e45ace6102 in 1290ms, sequenceid=287, compaction requested=false 2024-11-23T14:27:25,815 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3d5e64ff6f2c8eec1637e7e45ace6102: 2024-11-23T14:27:25,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-23T14:27:25,955 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-23T14:27:25,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:25,955 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2902): Flushing 3d5e64ff6f2c8eec1637e7e45ace6102 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-23T14:27:25,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/A/b0ffaadb79f1426a8b5efeb5f96713b8 is 50, key is test_row_0/A:col10/1732372044555/Put/seqid=0 2024-11-23T14:27:25,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742375_1551 (size=12301) 2024-11-23T14:27:26,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-23T14:27:26,364 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/A/b0ffaadb79f1426a8b5efeb5f96713b8 2024-11-23T14:27:26,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/B/b4be801b60634c5c8cb07bb0c54b3bc1 is 50, key is test_row_0/B:col10/1732372044555/Put/seqid=0 2024-11-23T14:27:26,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742376_1552 (size=12301) 2024-11-23T14:27:26,376 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/B/b4be801b60634c5c8cb07bb0c54b3bc1 2024-11-23T14:27:26,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/C/921f51a9fb984010b7523b10db3906b1 is 50, key is test_row_0/C:col10/1732372044555/Put/seqid=0 2024-11-23T14:27:26,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742377_1553 (size=12301) 2024-11-23T14:27:26,387 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/C/921f51a9fb984010b7523b10db3906b1 2024-11-23T14:27:26,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/A/b0ffaadb79f1426a8b5efeb5f96713b8 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/b0ffaadb79f1426a8b5efeb5f96713b8 2024-11-23T14:27:26,396 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/b0ffaadb79f1426a8b5efeb5f96713b8, entries=150, sequenceid=315, filesize=12.0 K 2024-11-23T14:27:26,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/B/b4be801b60634c5c8cb07bb0c54b3bc1 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/b4be801b60634c5c8cb07bb0c54b3bc1 2024-11-23T14:27:26,402 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/b4be801b60634c5c8cb07bb0c54b3bc1, entries=150, sequenceid=315, filesize=12.0 K 2024-11-23T14:27:26,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/C/921f51a9fb984010b7523b10db3906b1 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/921f51a9fb984010b7523b10db3906b1 2024-11-23T14:27:26,406 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/921f51a9fb984010b7523b10db3906b1, entries=150, sequenceid=315, filesize=12.0 K 2024-11-23T14:27:26,406 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=0 B/0 for 3d5e64ff6f2c8eec1637e7e45ace6102 in 451ms, sequenceid=315, compaction requested=true 2024-11-23T14:27:26,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2603): Flush status journal for 3d5e64ff6f2c8eec1637e7e45ace6102: 2024-11-23T14:27:26,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:26,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=140 2024-11-23T14:27:26,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster(4169): Remote procedure done, pid=140 2024-11-23T14:27:26,412 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-23T14:27:26,413 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2160 sec 2024-11-23T14:27:26,414 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees in 1.2220 sec 2024-11-23T14:27:26,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on 3d5e64ff6f2c8eec1637e7e45ace6102 2024-11-23T14:27:26,628 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T14:27:26,628 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3d5e64ff6f2c8eec1637e7e45ace6102 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-23T14:27:26,631 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/A/dd373392bb194755a0424491487749ba is 50, key is test_row_0/A:col10/1732372046626/Put/seqid=0 2024-11-23T14:27:26,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742378_1554 (size=14741) 2024-11-23T14:27:26,636 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/A/dd373392bb194755a0424491487749ba 2024-11-23T14:27:26,643 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/B/f072617d0f704b14b81cf2fe71ba8cd6 is 50, key is test_row_0/B:col10/1732372046626/Put/seqid=0 2024-11-23T14:27:26,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742379_1555 (size=12301) 2024-11-23T14:27:26,750 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:26,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32854 deadline: 1732372106739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:26,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:26,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32840 deadline: 1732372106740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:26,751 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:26,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:26,751 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:26,751 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:26,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32862 deadline: 1732372106741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:26,751 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:26,751 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:26,751 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:26,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:26,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32798 deadline: 1732372106743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:26,751 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:26,752 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:26,752 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:26,752 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:26,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32790 deadline: 1732372106745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:26,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:26,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:26,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:26,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:26,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:26,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:26,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:26,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32840 deadline: 1732372106860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:26,868 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:26,868 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:26,868 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:26,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32854 deadline: 1732372106860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:26,868 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:26,868 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:26,868 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:26,868 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:26,868 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:26,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32790 deadline: 1732372106860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:26,868 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:26,868 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:26,868 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:26,868 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:26,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32862 deadline: 1732372106860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:26,869 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:26,869 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:26,869 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:26,869 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:26,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32798 deadline: 1732372106860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:26,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:26,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:26,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:27,047 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/B/f072617d0f704b14b81cf2fe71ba8cd6 2024-11-23T14:27:27,059 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/C/d7113f3730854aa1a1896830369a8929 is 50, key is test_row_0/C:col10/1732372046626/Put/seqid=0 2024-11-23T14:27:27,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742380_1556 (size=12301) 2024-11-23T14:27:27,070 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/C/d7113f3730854aa1a1896830369a8929 2024-11-23T14:27:27,075 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/A/dd373392bb194755a0424491487749ba as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/dd373392bb194755a0424491487749ba 2024-11-23T14:27:27,079 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/dd373392bb194755a0424491487749ba, entries=200, sequenceid=326, filesize=14.4 K 2024-11-23T14:27:27,080 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/B/f072617d0f704b14b81cf2fe71ba8cd6 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/f072617d0f704b14b81cf2fe71ba8cd6 2024-11-23T14:27:27,082 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:27,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32854 deadline: 1732372107071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:27,084 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:27,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32840 deadline: 1732372107081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:27,084 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:27,084 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:27,084 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:27,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32790 deadline: 1732372107081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:27,084 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:27,084 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:27,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32862 deadline: 1732372107081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:27,085 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:27,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32798 deadline: 1732372107081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:27,085 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:27,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:27,085 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:27,085 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:27,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:27,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:27,085 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:27,085 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:27,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:27,085 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:27,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:27,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:27,088 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/f072617d0f704b14b81cf2fe71ba8cd6, entries=150, sequenceid=326, filesize=12.0 K 2024-11-23T14:27:27,089 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/C/d7113f3730854aa1a1896830369a8929 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/d7113f3730854aa1a1896830369a8929 2024-11-23T14:27:27,093 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/d7113f3730854aa1a1896830369a8929, entries=150, sequenceid=326, filesize=12.0 K 2024-11-23T14:27:27,094 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 3d5e64ff6f2c8eec1637e7e45ace6102 in 466ms, sequenceid=326, compaction requested=true 2024-11-23T14:27:27,094 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3d5e64ff6f2c8eec1637e7e45ace6102: 2024-11-23T14:27:27,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3d5e64ff6f2c8eec1637e7e45ace6102:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T14:27:27,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:27:27,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3d5e64ff6f2c8eec1637e7e45ace6102:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T14:27:27,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-23T14:27:27,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3d5e64ff6f2c8eec1637e7e45ace6102:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T14:27:27,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-23T14:27:27,095 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T14:27:27,095 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T14:27:27,097 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54732 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T14:27:27,097 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1541): 3d5e64ff6f2c8eec1637e7e45ace6102/A is initiating minor compaction (all files) 2024-11-23T14:27:27,097 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3d5e64ff6f2c8eec1637e7e45ace6102/A in TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:27,097 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/9a9b4454ed0148169e49e39e2a7b2c8b, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/05f3fddd57be4890a9e51984caf38eee, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/b0ffaadb79f1426a8b5efeb5f96713b8, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/dd373392bb194755a0424491487749ba] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp, totalSize=53.4 K 2024-11-23T14:27:27,098 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T14:27:27,098 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1541): 3d5e64ff6f2c8eec1637e7e45ace6102/C is initiating minor compaction (all files) 2024-11-23T14:27:27,098 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3d5e64ff6f2c8eec1637e7e45ace6102/C in TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:27,098 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/dedd24e335774be5a3ca6ab2398383ba, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/7dd5e0b989cc42a884e5f843fe93dbe1, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/921f51a9fb984010b7523b10db3906b1, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/d7113f3730854aa1a1896830369a8929] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp, totalSize=48.7 K 2024-11-23T14:27:27,098 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9a9b4454ed0148169e49e39e2a7b2c8b, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1732372042210 2024-11-23T14:27:27,098 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting dedd24e335774be5a3ca6ab2398383ba, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1732372042210 2024-11-23T14:27:27,099 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05f3fddd57be4890a9e51984caf38eee, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732372043370 2024-11-23T14:27:27,099 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 7dd5e0b989cc42a884e5f843fe93dbe1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732372043370 2024-11-23T14:27:27,099 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting b0ffaadb79f1426a8b5efeb5f96713b8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1732372044555 2024-11-23T14:27:27,099 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 921f51a9fb984010b7523b10db3906b1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1732372044555 2024-11-23T14:27:27,099 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd373392bb194755a0424491487749ba, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1732372046607 2024-11-23T14:27:27,099 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting d7113f3730854aa1a1896830369a8929, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1732372046611 2024-11-23T14:27:27,106 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3d5e64ff6f2c8eec1637e7e45ace6102#A#compaction#476 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:27:27,107 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/A/0d882d1cf0744226a4c891976cdc4a0e is 50, key is test_row_0/A:col10/1732372046626/Put/seqid=0 2024-11-23T14:27:27,110 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3d5e64ff6f2c8eec1637e7e45ace6102#C#compaction#477 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:27:27,111 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/C/45d3cc28fb6a4df68b174fa1c2bb0956 is 50, key is test_row_0/C:col10/1732372046626/Put/seqid=0 2024-11-23T14:27:27,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742381_1557 (size=13085) 2024-11-23T14:27:27,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742382_1558 (size=13085) 2024-11-23T14:27:27,145 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/C/45d3cc28fb6a4df68b174fa1c2bb0956 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/45d3cc28fb6a4df68b174fa1c2bb0956 2024-11-23T14:27:27,152 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3d5e64ff6f2c8eec1637e7e45ace6102/C of 3d5e64ff6f2c8eec1637e7e45ace6102 into 45d3cc28fb6a4df68b174fa1c2bb0956(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:27:27,152 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3d5e64ff6f2c8eec1637e7e45ace6102: 2024-11-23T14:27:27,152 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., storeName=3d5e64ff6f2c8eec1637e7e45ace6102/C, priority=12, startTime=1732372047094; duration=0sec 2024-11-23T14:27:27,152 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:27:27,152 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3d5e64ff6f2c8eec1637e7e45ace6102:C 2024-11-23T14:27:27,153 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T14:27:27,153 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T14:27:27,153 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1541): 3d5e64ff6f2c8eec1637e7e45ace6102/B is initiating minor compaction (all files) 2024-11-23T14:27:27,154 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3d5e64ff6f2c8eec1637e7e45ace6102/B in TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:27,154 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/ea1adfc7e6d74baa8d8a4998bb182be2, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/ebba26bc23b44225aad1cfbd024e8957, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/b4be801b60634c5c8cb07bb0c54b3bc1, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/f072617d0f704b14b81cf2fe71ba8cd6] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp, totalSize=48.7 K 2024-11-23T14:27:27,156 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting ea1adfc7e6d74baa8d8a4998bb182be2, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1732372042210 2024-11-23T14:27:27,158 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting ebba26bc23b44225aad1cfbd024e8957, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732372043370 2024-11-23T14:27:27,158 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting b4be801b60634c5c8cb07bb0c54b3bc1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1732372044555 2024-11-23T14:27:27,158 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting f072617d0f704b14b81cf2fe71ba8cd6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1732372046611 2024-11-23T14:27:27,164 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3d5e64ff6f2c8eec1637e7e45ace6102#B#compaction#478 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:27:27,165 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/B/772a18f207bd4da394e3f8a207a9dbf0 is 50, key is test_row_0/B:col10/1732372046626/Put/seqid=0 2024-11-23T14:27:27,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742383_1559 (size=13085) 2024-11-23T14:27:27,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-23T14:27:27,350 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T14:27:27,351 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-23T14:27:27,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees 2024-11-23T14:27:27,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-23T14:27:27,352 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=141, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T14:27:27,353 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=141, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T14:27:27,353 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T14:27:27,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on 3d5e64ff6f2c8eec1637e7e45ace6102 2024-11-23T14:27:27,397 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T14:27:27,397 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3d5e64ff6f2c8eec1637e7e45ace6102 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-23T14:27:27,402 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/A/d11b2a0645fe49d2bbe15695b9ce656b is 50, key is test_row_0/A:col10/1732372047396/Put/seqid=0 2024-11-23T14:27:27,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742384_1560 (size=12301) 2024-11-23T14:27:27,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:27,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32790 deadline: 1732372107400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:27,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:27,410 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:27,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32862 deadline: 1732372107400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:27,410 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:27,410 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:27,410 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:27,410 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:27,410 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:27,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:27,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32854 deadline: 1732372107408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:27,412 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:27,412 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:27,412 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:27,419 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:27,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32798 deadline: 1732372107409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:27,420 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:27,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32840 deadline: 1732372107410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:27,420 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:27,420 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:27,420 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:27,420 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:27,420 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:27,420 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:27,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-23T14:27:27,505 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-23T14:27:27,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:27,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. as already flushing 2024-11-23T14:27:27,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:27,505 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:27,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:27,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:27,524 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/A/0d882d1cf0744226a4c891976cdc4a0e as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/0d882d1cf0744226a4c891976cdc4a0e 2024-11-23T14:27:27,524 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:27,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32854 deadline: 1732372107520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:27,525 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:27,525 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:27,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32862 deadline: 1732372107520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:27,525 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:27,525 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:27,525 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:27,525 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:27,525 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:27,525 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:27,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32790 deadline: 1732372107522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:27,525 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:27,526 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:27,526 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:27,528 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3d5e64ff6f2c8eec1637e7e45ace6102/A of 3d5e64ff6f2c8eec1637e7e45ace6102 into 0d882d1cf0744226a4c891976cdc4a0e(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:27:27,528 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3d5e64ff6f2c8eec1637e7e45ace6102: 2024-11-23T14:27:27,528 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., storeName=3d5e64ff6f2c8eec1637e7e45ace6102/A, priority=12, startTime=1732372047094; duration=0sec 2024-11-23T14:27:27,528 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:27:27,528 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3d5e64ff6f2c8eec1637e7e45ace6102:A 2024-11-23T14:27:27,537 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:27,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32798 deadline: 1732372107530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:27,537 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:27,537 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:27,537 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:27,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32840 deadline: 1732372107530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:27,537 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:27,537 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:27,537 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:27,537 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:27,581 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/B/772a18f207bd4da394e3f8a207a9dbf0 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/772a18f207bd4da394e3f8a207a9dbf0 2024-11-23T14:27:27,588 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3d5e64ff6f2c8eec1637e7e45ace6102/B of 3d5e64ff6f2c8eec1637e7e45ace6102 into 772a18f207bd4da394e3f8a207a9dbf0(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:27:27,588 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3d5e64ff6f2c8eec1637e7e45ace6102: 2024-11-23T14:27:27,588 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., storeName=3d5e64ff6f2c8eec1637e7e45ace6102/B, priority=12, startTime=1732372047094; duration=0sec 2024-11-23T14:27:27,589 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:27:27,589 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3d5e64ff6f2c8eec1637e7e45ace6102:B 2024-11-23T14:27:27,659 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-23T14:27:27,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:27,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. as already flushing 2024-11-23T14:27:27,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:27,659 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:27,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:27,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:27,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-23T14:27:27,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:27,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32854 deadline: 1732372107730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:27,736 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:27,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32862 deadline: 1732372107730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:27,736 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:27,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32790 deadline: 1732372107730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:27,736 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:27,736 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:27,736 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:27,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:27,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:27,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:27,737 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:27,737 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:27,737 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:27,745 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:27,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32840 deadline: 1732372107741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:27,745 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:27,745 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:27,745 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:27,746 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:27,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32798 deadline: 1732372107743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:27,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:27,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:27,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:27,805 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/A/d11b2a0645fe49d2bbe15695b9ce656b 2024-11-23T14:27:27,812 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/B/7a4785ba6c9641649fbcd1dcc0ee4d04 is 50, key is test_row_0/B:col10/1732372047396/Put/seqid=0 2024-11-23T14:27:27,813 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-23T14:27:27,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:27,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. as already flushing 2024-11-23T14:27:27,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:27,813 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:27,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:27,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:27,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742385_1561 (size=12301) 2024-11-23T14:27:27,816 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/B/7a4785ba6c9641649fbcd1dcc0ee4d04 2024-11-23T14:27:27,824 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/C/8f60fd452c564c7a9085649cfbaf8f55 is 50, key is test_row_0/C:col10/1732372047396/Put/seqid=0 2024-11-23T14:27:27,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742386_1562 (size=12301) 2024-11-23T14:27:27,830 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/C/8f60fd452c564c7a9085649cfbaf8f55 2024-11-23T14:27:27,833 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/A/d11b2a0645fe49d2bbe15695b9ce656b as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/d11b2a0645fe49d2bbe15695b9ce656b 2024-11-23T14:27:27,837 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/d11b2a0645fe49d2bbe15695b9ce656b, entries=150, sequenceid=353, filesize=12.0 K 2024-11-23T14:27:27,837 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/B/7a4785ba6c9641649fbcd1dcc0ee4d04 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/7a4785ba6c9641649fbcd1dcc0ee4d04 2024-11-23T14:27:27,841 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/7a4785ba6c9641649fbcd1dcc0ee4d04, entries=150, sequenceid=353, filesize=12.0 K 2024-11-23T14:27:27,841 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-23T14:27:27,844 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/C/8f60fd452c564c7a9085649cfbaf8f55 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/8f60fd452c564c7a9085649cfbaf8f55 2024-11-23T14:27:27,848 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/8f60fd452c564c7a9085649cfbaf8f55, entries=150, sequenceid=353, filesize=12.0 K 2024-11-23T14:27:27,849 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 3d5e64ff6f2c8eec1637e7e45ace6102 in 452ms, sequenceid=353, compaction requested=false 2024-11-23T14:27:27,849 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3d5e64ff6f2c8eec1637e7e45ace6102: 2024-11-23T14:27:27,967 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-23T14:27:27,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:27,968 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2902): Flushing 3d5e64ff6f2c8eec1637e7e45ace6102 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-23T14:27:27,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/A/38ef3161ad5643629f36ea8d7ff5149e is 50, key is test_row_0/A:col10/1732372047399/Put/seqid=0 2024-11-23T14:27:27,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-23T14:27:28,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742387_1563 (size=12301) 2024-11-23T14:27:28,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on 3d5e64ff6f2c8eec1637e7e45ace6102 2024-11-23T14:27:28,089 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. as already flushing 2024-11-23T14:27:28,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:28,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32790 deadline: 1732372108151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:28,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:28,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:28,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:28,182 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:28,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32798 deadline: 1732372108166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:28,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:28,183 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:28,183 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:28,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32862 deadline: 1732372108166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:28,183 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:28,183 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:28,183 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:28,183 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:28,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:28,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32854 deadline: 1732372108168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:28,184 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:28,184 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:28,184 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:28,184 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:28,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32840 deadline: 1732372108168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:28,184 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:28,184 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:28,184 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:28,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:28,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32790 deadline: 1732372108269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:28,275 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:28,275 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:28,275 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:28,277 INFO [Thread-2179 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2179. 2024-11-23T14:27:28,277 DEBUG [Thread-2179 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T14:27:28,277 DEBUG [Thread-2179 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:27:28,277 DEBUG [Thread-2179 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:27:28,277 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T14:27:28,280 INFO [Thread-2185 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2185. 2024-11-23T14:27:28,280 DEBUG [Thread-2185 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T14:27:28,280 DEBUG [Thread-2185 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:27:28,280 DEBUG [Thread-2185 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:27:28,280 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T14:27:28,282 INFO [Thread-2183 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2183. 2024-11-23T14:27:28,282 DEBUG [Thread-2183 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T14:27:28,282 DEBUG [Thread-2183 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:27:28,282 DEBUG [Thread-2183 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:27:28,282 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T14:27:28,286 INFO [Thread-2181 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2181. 2024-11-23T14:27:28,286 DEBUG [Thread-2181 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T14:27:28,286 DEBUG [Thread-2181 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:27:28,287 DEBUG [Thread-2181 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:27:28,287 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T14:27:28,287 INFO [Thread-2177 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2177. 2024-11-23T14:27:28,287 DEBUG [Thread-2177 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T14:27:28,287 DEBUG [Thread-2177 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:27:28,287 DEBUG [Thread-2177 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:27:28,287 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T14:27:28,290 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:28,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:28,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:28,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32854 deadline: 1732372108290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:28,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32862 deadline: 1732372108290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:28,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32798 deadline: 1732372108290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:28,290 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:28,290 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:28,290 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:28,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:28,290 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:28,290 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:28,291 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:28,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:28,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32840 deadline: 1732372108290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:28,291 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:28,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:28,291 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:28,291 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:28,291 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:28,404 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/A/38ef3161ad5643629f36ea8d7ff5149e 2024-11-23T14:27:28,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/B/1bd6d986a4d54eb1985f37cf6d68faef is 50, key is test_row_0/B:col10/1732372047399/Put/seqid=0 2024-11-23T14:27:28,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742388_1564 (size=12301) 2024-11-23T14:27:28,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:28,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32790 deadline: 1732372108480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:28,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:28,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:28,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:28,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-23T14:27:28,501 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:28,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32854 deadline: 1732372108501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:28,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:28,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32862 deadline: 1732372108500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:28,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:28,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32798 deadline: 1732372108501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:28,502 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:28,502 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:28,502 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:28,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:28,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:28,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:28,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:28,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32840 deadline: 1732372108501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:28,502 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:28,502 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:28,502 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:28,502 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:28,502 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:28,502 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:28,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:28,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32790 deadline: 1732372108790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:28,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:28,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:28,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:28,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:28,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:28,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32854 deadline: 1732372108810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:28,810 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:28,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32798 deadline: 1732372108810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:28,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32862 deadline: 1732372108810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:28,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:28,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:32840 deadline: 1732372108810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:28,810 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:28,810 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:28,810 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:28,810 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:28,810 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:28,810 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:28,811 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:28,811 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:28,811 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:28,811 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 , the old value is region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:28,811 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:28,811 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., hostname=c6f7b1897123,35385,1732371902767, seqNum=2 because the exception is null or not the one we care about 2024-11-23T14:27:28,813 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/B/1bd6d986a4d54eb1985f37cf6d68faef 2024-11-23T14:27:28,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/C/04f24fbd06cd49d58a5e1d5b9bdfa24c is 50, key is test_row_0/C:col10/1732372047399/Put/seqid=0 2024-11-23T14:27:28,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742389_1565 (size=12301) 2024-11-23T14:27:29,222 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/C/04f24fbd06cd49d58a5e1d5b9bdfa24c 2024-11-23T14:27:29,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/A/38ef3161ad5643629f36ea8d7ff5149e as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/38ef3161ad5643629f36ea8d7ff5149e 2024-11-23T14:27:29,230 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/38ef3161ad5643629f36ea8d7ff5149e, entries=150, sequenceid=365, filesize=12.0 K 2024-11-23T14:27:29,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/B/1bd6d986a4d54eb1985f37cf6d68faef as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/1bd6d986a4d54eb1985f37cf6d68faef 2024-11-23T14:27:29,234 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/1bd6d986a4d54eb1985f37cf6d68faef, entries=150, sequenceid=365, filesize=12.0 K 2024-11-23T14:27:29,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/C/04f24fbd06cd49d58a5e1d5b9bdfa24c as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/04f24fbd06cd49d58a5e1d5b9bdfa24c 2024-11-23T14:27:29,238 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/04f24fbd06cd49d58a5e1d5b9bdfa24c, entries=150, sequenceid=365, filesize=12.0 K 2024-11-23T14:27:29,239 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=161.02 KB/164880 for 3d5e64ff6f2c8eec1637e7e45ace6102 in 1272ms, sequenceid=365, compaction requested=true 2024-11-23T14:27:29,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2603): Flush status journal for 3d5e64ff6f2c8eec1637e7e45ace6102: 2024-11-23T14:27:29,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:29,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=142 2024-11-23T14:27:29,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster(4169): Remote procedure done, pid=142 2024-11-23T14:27:29,242 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-23T14:27:29,243 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8880 sec 2024-11-23T14:27:29,245 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees in 1.8920 sec 2024-11-23T14:27:29,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on 3d5e64ff6f2c8eec1637e7e45ace6102 2024-11-23T14:27:29,302 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T14:27:29,302 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3d5e64ff6f2c8eec1637e7e45ace6102 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-23T14:27:29,302 INFO [Thread-2174 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2174. 2024-11-23T14:27:29,302 DEBUG [Thread-2174 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T14:27:29,303 DEBUG [Thread-2174 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:27:29,303 DEBUG [Thread-2174 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:27:29,303 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T14:27:29,307 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/A/e87e0f7d0fe146e18f40397bf91dc1c1 is 50, key is test_row_0/A:col10/1732372048146/Put/seqid=0 2024-11-23T14:27:29,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742390_1566 (size=12301) 2024-11-23T14:27:29,323 INFO [Thread-2166 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2166. 2024-11-23T14:27:29,323 INFO [Thread-2172 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2172. 2024-11-23T14:27:29,323 DEBUG [Thread-2166 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T14:27:29,323 DEBUG [Thread-2172 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T14:27:29,323 DEBUG [Thread-2166 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:27:29,323 DEBUG [Thread-2172 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:27:29,323 INFO [Thread-2170 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2170. 2024-11-23T14:27:29,323 DEBUG [Thread-2170 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T14:27:29,323 DEBUG [Thread-2172 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:27:29,323 DEBUG [Thread-2166 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:27:29,323 DEBUG [Thread-2170 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:27:29,323 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T14:27:29,323 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T14:27:29,323 DEBUG [Thread-2170 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:27:29,323 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T14:27:29,324 INFO [Thread-2168 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2168. 2024-11-23T14:27:29,324 DEBUG [Thread-2168 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T14:27:29,324 DEBUG [Thread-2168 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:27:29,324 DEBUG [Thread-2168 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:27:29,325 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T14:27:29,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-23T14:27:29,501 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T14:27:29,501 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-23T14:27:29,501 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 67 2024-11-23T14:27:29,501 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 39 2024-11-23T14:27:29,501 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 71 2024-11-23T14:27:29,501 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 71 2024-11-23T14:27:29,501 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 67 2024-11-23T14:27:29,501 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-23T14:27:29,501 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-23T14:27:29,501 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1723 2024-11-23T14:27:29,501 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5166 rows 2024-11-23T14:27:29,501 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1719 2024-11-23T14:27:29,501 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5157 rows 2024-11-23T14:27:29,501 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1724 2024-11-23T14:27:29,501 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5172 rows 2024-11-23T14:27:29,501 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1719 2024-11-23T14:27:29,501 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5157 rows 2024-11-23T14:27:29,501 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1726 2024-11-23T14:27:29,501 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5177 rows 2024-11-23T14:27:29,501 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-23T14:27:29,502 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T14:27:29,502 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:27:29,502 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:27:29,502 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T14:27:29,502 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-23T14:27:29,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] procedure2.ProcedureExecutor(1139): Stored pid=143, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-23T14:27:29,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-23T14:27:29,511 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732372049511"}]},"ts":"1732372049511"} 2024-11-23T14:27:29,513 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-23T14:27:29,513 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-23T14:27:29,514 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-23T14:27:29,516 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3d5e64ff6f2c8eec1637e7e45ace6102, UNASSIGN}] 2024-11-23T14:27:29,517 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3d5e64ff6f2c8eec1637e7e45ace6102, UNASSIGN 2024-11-23T14:27:29,517 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=3d5e64ff6f2c8eec1637e7e45ace6102, regionState=CLOSING, regionLocation=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:29,519 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3d5e64ff6f2c8eec1637e7e45ace6102, UNASSIGN because future has completed 2024-11-23T14:27:29,519 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-23T14:27:29,519 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767}] 2024-11-23T14:27:29,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-23T14:27:29,672 INFO [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] handler.UnassignRegionHandler(122): Close 3d5e64ff6f2c8eec1637e7e45ace6102 2024-11-23T14:27:29,672 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-23T14:27:29,672 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1722): Closing 3d5e64ff6f2c8eec1637e7e45ace6102, disabling compactions & flushes 2024-11-23T14:27:29,672 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:29,713 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/A/e87e0f7d0fe146e18f40397bf91dc1c1 2024-11-23T14:27:29,720 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/B/dfa96b483d18484ab7402b7ebaf63e08 is 50, key is test_row_0/B:col10/1732372048146/Put/seqid=0 2024-11-23T14:27:29,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742391_1567 (size=12301) 2024-11-23T14:27:29,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-23T14:27:30,124 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/B/dfa96b483d18484ab7402b7ebaf63e08 2024-11-23T14:27:30,130 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/C/44f0c5fdbb8f4ed7b9c462a49f086901 is 50, key is test_row_0/C:col10/1732372048146/Put/seqid=0 2024-11-23T14:27:30,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742392_1568 (size=12301) 2024-11-23T14:27:30,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-23T14:27:30,533 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/C/44f0c5fdbb8f4ed7b9c462a49f086901 2024-11-23T14:27:30,536 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/A/e87e0f7d0fe146e18f40397bf91dc1c1 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/e87e0f7d0fe146e18f40397bf91dc1c1 2024-11-23T14:27:30,539 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/e87e0f7d0fe146e18f40397bf91dc1c1, entries=150, sequenceid=393, filesize=12.0 K 2024-11-23T14:27:30,539 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/B/dfa96b483d18484ab7402b7ebaf63e08 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/dfa96b483d18484ab7402b7ebaf63e08 2024-11-23T14:27:30,542 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/dfa96b483d18484ab7402b7ebaf63e08, entries=150, sequenceid=393, filesize=12.0 K 2024-11-23T14:27:30,542 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/C/44f0c5fdbb8f4ed7b9c462a49f086901 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/44f0c5fdbb8f4ed7b9c462a49f086901 2024-11-23T14:27:30,545 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/44f0c5fdbb8f4ed7b9c462a49f086901, entries=150, sequenceid=393, filesize=12.0 K 2024-11-23T14:27:30,546 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=26.84 KB/27480 for 3d5e64ff6f2c8eec1637e7e45ace6102 in 1243ms, sequenceid=393, compaction requested=true 2024-11-23T14:27:30,546 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3d5e64ff6f2c8eec1637e7e45ace6102: 2024-11-23T14:27:30,546 INFO [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:30,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3d5e64ff6f2c8eec1637e7e45ace6102:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T14:27:30,546 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. because compaction request was cancelled 2024-11-23T14:27:30,546 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:30,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:27:30,546 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3d5e64ff6f2c8eec1637e7e45ace6102:A 2024-11-23T14:27:30,546 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. after waiting 0 ms 2024-11-23T14:27:30,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3d5e64ff6f2c8eec1637e7e45ace6102:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T14:27:30,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:27:30,546 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. because compaction request was cancelled 2024-11-23T14:27:30,546 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:30,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3d5e64ff6f2c8eec1637e7e45ace6102:C, priority=-2147483648, current under compaction store size is 2 2024-11-23T14:27:30,546 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3d5e64ff6f2c8eec1637e7e45ace6102:B 2024-11-23T14:27:30,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:27:30,546 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. because compaction request was cancelled 2024-11-23T14:27:30,546 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3d5e64ff6f2c8eec1637e7e45ace6102:C 2024-11-23T14:27:30,546 INFO [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(2902): Flushing 3d5e64ff6f2c8eec1637e7e45ace6102 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-23T14:27:30,549 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/A/09b737907d7e42a9b14adbf0306549a2 is 50, key is test_row_0/A:col10/1732372049322/Put/seqid=0 2024-11-23T14:27:30,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742393_1569 (size=9857) 2024-11-23T14:27:30,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-23T14:27:30,798 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-23T14:27:30,953 INFO [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/A/09b737907d7e42a9b14adbf0306549a2 2024-11-23T14:27:30,960 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/B/6acb234546204fe793f1208acf0ca4cd is 50, key is test_row_0/B:col10/1732372049322/Put/seqid=0 2024-11-23T14:27:30,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742394_1570 (size=9857) 2024-11-23T14:27:31,363 INFO [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/B/6acb234546204fe793f1208acf0ca4cd 2024-11-23T14:27:31,369 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/C/4ba61b8127b44094a71cdab0da5f0117 is 50, key is test_row_0/C:col10/1732372049322/Put/seqid=0 2024-11-23T14:27:31,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742395_1571 (size=9857) 2024-11-23T14:27:31,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-23T14:27:31,772 INFO [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/C/4ba61b8127b44094a71cdab0da5f0117 2024-11-23T14:27:31,775 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/A/09b737907d7e42a9b14adbf0306549a2 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/09b737907d7e42a9b14adbf0306549a2 2024-11-23T14:27:31,777 INFO [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/09b737907d7e42a9b14adbf0306549a2, entries=100, sequenceid=400, filesize=9.6 K 2024-11-23T14:27:31,778 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/B/6acb234546204fe793f1208acf0ca4cd as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/6acb234546204fe793f1208acf0ca4cd 2024-11-23T14:27:31,781 INFO [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/6acb234546204fe793f1208acf0ca4cd, entries=100, sequenceid=400, filesize=9.6 K 2024-11-23T14:27:31,781 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/.tmp/C/4ba61b8127b44094a71cdab0da5f0117 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/4ba61b8127b44094a71cdab0da5f0117 2024-11-23T14:27:31,784 INFO [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/4ba61b8127b44094a71cdab0da5f0117, entries=100, sequenceid=400, filesize=9.6 K 2024-11-23T14:27:31,785 INFO [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 3d5e64ff6f2c8eec1637e7e45ace6102 in 1239ms, sequenceid=400, compaction requested=true 2024-11-23T14:27:31,785 DEBUG [StoreCloser-TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/4326dcf71488476781a63ed39c584cb6, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/2cd3002d22d04490bcf64f7600b3645c, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/405d6d89a67d4480905356ee2054dc33, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/e64a4392a9434152bbbc862a3648d9c5, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/be523536339b46609d06d999bb0f64bf, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/b36eb1221c1a43ca856d4482b40ef868, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/aac3f99378fc48bab42bb84d214f64a8, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/688a7150665d482d9513a16f3f0835c9, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/735cdb09a36b4058b8aac96bc5149d10, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/93b311ee242c4765b199e56aabe970c3, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/63e9792bae4341279862609258c1be04, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/24801b89b8a742adbaf0d5f240dcb832, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/fe3ae34564384756a95b94a414369998, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/4c2b779d046048db98334ce7feca0557, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/86154e038ddd4aeb8fb7dd60711884f7, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/f72b08d40b9f48268d1641eeedcf5d11, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/94debe76664643cd9d094eda878b5d2b, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/1930e6d09ca84f0da81540256f573090, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/7c364d794aa44313985efd9a49f6cd65, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/9a9b4454ed0148169e49e39e2a7b2c8b, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/05f3fddd57be4890a9e51984caf38eee, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/b0ffaadb79f1426a8b5efeb5f96713b8, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/dd373392bb194755a0424491487749ba] to archive 2024-11-23T14:27:31,786 DEBUG [StoreCloser-TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-23T14:27:31,787 DEBUG [StoreCloser-TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/4326dcf71488476781a63ed39c584cb6 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/4326dcf71488476781a63ed39c584cb6 2024-11-23T14:27:31,788 DEBUG [StoreCloser-TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/2cd3002d22d04490bcf64f7600b3645c to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/2cd3002d22d04490bcf64f7600b3645c 2024-11-23T14:27:31,789 DEBUG [StoreCloser-TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/405d6d89a67d4480905356ee2054dc33 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/405d6d89a67d4480905356ee2054dc33 2024-11-23T14:27:31,789 DEBUG [StoreCloser-TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/e64a4392a9434152bbbc862a3648d9c5 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/e64a4392a9434152bbbc862a3648d9c5 2024-11-23T14:27:31,790 DEBUG [StoreCloser-TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/be523536339b46609d06d999bb0f64bf to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/be523536339b46609d06d999bb0f64bf 2024-11-23T14:27:31,791 DEBUG [StoreCloser-TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/b36eb1221c1a43ca856d4482b40ef868 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/b36eb1221c1a43ca856d4482b40ef868 2024-11-23T14:27:31,792 DEBUG [StoreCloser-TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/aac3f99378fc48bab42bb84d214f64a8 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/aac3f99378fc48bab42bb84d214f64a8 2024-11-23T14:27:31,792 DEBUG [StoreCloser-TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/688a7150665d482d9513a16f3f0835c9 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/688a7150665d482d9513a16f3f0835c9 2024-11-23T14:27:31,793 DEBUG [StoreCloser-TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/735cdb09a36b4058b8aac96bc5149d10 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/735cdb09a36b4058b8aac96bc5149d10 2024-11-23T14:27:31,794 DEBUG [StoreCloser-TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/93b311ee242c4765b199e56aabe970c3 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/93b311ee242c4765b199e56aabe970c3 2024-11-23T14:27:31,795 DEBUG [StoreCloser-TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/63e9792bae4341279862609258c1be04 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/63e9792bae4341279862609258c1be04 2024-11-23T14:27:31,796 DEBUG [StoreCloser-TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/24801b89b8a742adbaf0d5f240dcb832 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/24801b89b8a742adbaf0d5f240dcb832 2024-11-23T14:27:31,796 DEBUG [StoreCloser-TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/fe3ae34564384756a95b94a414369998 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/fe3ae34564384756a95b94a414369998 2024-11-23T14:27:31,797 DEBUG [StoreCloser-TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/4c2b779d046048db98334ce7feca0557 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/4c2b779d046048db98334ce7feca0557 2024-11-23T14:27:31,798 DEBUG [StoreCloser-TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/86154e038ddd4aeb8fb7dd60711884f7 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/86154e038ddd4aeb8fb7dd60711884f7 2024-11-23T14:27:31,798 DEBUG [StoreCloser-TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/f72b08d40b9f48268d1641eeedcf5d11 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/f72b08d40b9f48268d1641eeedcf5d11 2024-11-23T14:27:31,799 DEBUG [StoreCloser-TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/94debe76664643cd9d094eda878b5d2b to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/94debe76664643cd9d094eda878b5d2b 2024-11-23T14:27:31,800 DEBUG [StoreCloser-TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/1930e6d09ca84f0da81540256f573090 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/1930e6d09ca84f0da81540256f573090 2024-11-23T14:27:31,800 DEBUG [StoreCloser-TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/7c364d794aa44313985efd9a49f6cd65 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/7c364d794aa44313985efd9a49f6cd65 2024-11-23T14:27:31,801 DEBUG [StoreCloser-TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/9a9b4454ed0148169e49e39e2a7b2c8b to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/9a9b4454ed0148169e49e39e2a7b2c8b 2024-11-23T14:27:31,802 DEBUG [StoreCloser-TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/05f3fddd57be4890a9e51984caf38eee to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/05f3fddd57be4890a9e51984caf38eee 2024-11-23T14:27:31,802 DEBUG [StoreCloser-TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/b0ffaadb79f1426a8b5efeb5f96713b8 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/b0ffaadb79f1426a8b5efeb5f96713b8 2024-11-23T14:27:31,803 DEBUG [StoreCloser-TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/dd373392bb194755a0424491487749ba to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/dd373392bb194755a0424491487749ba 2024-11-23T14:27:31,804 DEBUG [StoreCloser-TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/5e21501f70d84a91a20ee58ddcad83cc, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/39d879c2832b44408f2b7b843fb78c67, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/cab0515b1ee241e8934188a6362891a0, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/7f63d5a736c143aaa83140b08517eda8, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/a4ddab881bdb4174bd1d6020c31b8d1f, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/9de94979b4ca42ab801ce8cdae9fa37c, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/4ecf371c663348a3bef6a539797ed0e2, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/12a20af8407545f8a808daf3c3a5941e, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/b0503383ed8644e78aeb8dcb01046183, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/c486d1ec51ad46579fc29e4a4cbe47b8, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/24817314092444f687176877654e99c4, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/0955be0a156a4974861e7fa0a3eb63d2, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/29e7efb03cc045ea909e2125c7659438, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/952d1fcdb09242f693c6c5342aabc19b, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/904f1b4f71bf4b3394ac1e1befc43a6d, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/fbf0fdd76e6d485799ca8eceeb9c3bf9, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/f3dfb676b57f41b19915a74662fc3e75, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/70cd4709c55d485b9d7f246f586a7206, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/ea1adfc7e6d74baa8d8a4998bb182be2, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/19f0b68ada014d11a2f9da277017385a, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/ebba26bc23b44225aad1cfbd024e8957, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/b4be801b60634c5c8cb07bb0c54b3bc1, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/f072617d0f704b14b81cf2fe71ba8cd6] to archive 2024-11-23T14:27:31,805 DEBUG [StoreCloser-TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-23T14:27:31,806 DEBUG [StoreCloser-TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/5e21501f70d84a91a20ee58ddcad83cc to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/5e21501f70d84a91a20ee58ddcad83cc 2024-11-23T14:27:31,807 DEBUG [StoreCloser-TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/39d879c2832b44408f2b7b843fb78c67 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/39d879c2832b44408f2b7b843fb78c67 2024-11-23T14:27:31,808 DEBUG [StoreCloser-TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/cab0515b1ee241e8934188a6362891a0 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/cab0515b1ee241e8934188a6362891a0 2024-11-23T14:27:31,808 DEBUG [StoreCloser-TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/7f63d5a736c143aaa83140b08517eda8 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/7f63d5a736c143aaa83140b08517eda8 2024-11-23T14:27:31,809 DEBUG [StoreCloser-TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/a4ddab881bdb4174bd1d6020c31b8d1f to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/a4ddab881bdb4174bd1d6020c31b8d1f 2024-11-23T14:27:31,810 DEBUG [StoreCloser-TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/9de94979b4ca42ab801ce8cdae9fa37c to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/9de94979b4ca42ab801ce8cdae9fa37c 2024-11-23T14:27:31,811 DEBUG [StoreCloser-TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/4ecf371c663348a3bef6a539797ed0e2 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/4ecf371c663348a3bef6a539797ed0e2 2024-11-23T14:27:31,811 DEBUG [StoreCloser-TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/12a20af8407545f8a808daf3c3a5941e to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/12a20af8407545f8a808daf3c3a5941e 2024-11-23T14:27:31,812 DEBUG [StoreCloser-TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/b0503383ed8644e78aeb8dcb01046183 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/b0503383ed8644e78aeb8dcb01046183 2024-11-23T14:27:31,813 DEBUG [StoreCloser-TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/c486d1ec51ad46579fc29e4a4cbe47b8 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/c486d1ec51ad46579fc29e4a4cbe47b8 2024-11-23T14:27:31,814 DEBUG [StoreCloser-TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/24817314092444f687176877654e99c4 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/24817314092444f687176877654e99c4 2024-11-23T14:27:31,814 DEBUG [StoreCloser-TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/0955be0a156a4974861e7fa0a3eb63d2 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/0955be0a156a4974861e7fa0a3eb63d2 2024-11-23T14:27:31,815 DEBUG [StoreCloser-TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/29e7efb03cc045ea909e2125c7659438 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/29e7efb03cc045ea909e2125c7659438 2024-11-23T14:27:31,816 DEBUG [StoreCloser-TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/952d1fcdb09242f693c6c5342aabc19b to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/952d1fcdb09242f693c6c5342aabc19b 2024-11-23T14:27:31,817 DEBUG [StoreCloser-TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/904f1b4f71bf4b3394ac1e1befc43a6d to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/904f1b4f71bf4b3394ac1e1befc43a6d 2024-11-23T14:27:31,817 DEBUG [StoreCloser-TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/fbf0fdd76e6d485799ca8eceeb9c3bf9 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/fbf0fdd76e6d485799ca8eceeb9c3bf9 2024-11-23T14:27:31,818 DEBUG [StoreCloser-TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/f3dfb676b57f41b19915a74662fc3e75 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/f3dfb676b57f41b19915a74662fc3e75 2024-11-23T14:27:31,819 DEBUG [StoreCloser-TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/70cd4709c55d485b9d7f246f586a7206 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/70cd4709c55d485b9d7f246f586a7206 2024-11-23T14:27:31,820 DEBUG [StoreCloser-TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/ea1adfc7e6d74baa8d8a4998bb182be2 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/ea1adfc7e6d74baa8d8a4998bb182be2 2024-11-23T14:27:31,821 DEBUG [StoreCloser-TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/19f0b68ada014d11a2f9da277017385a to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/19f0b68ada014d11a2f9da277017385a 2024-11-23T14:27:31,821 DEBUG [StoreCloser-TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/ebba26bc23b44225aad1cfbd024e8957 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/ebba26bc23b44225aad1cfbd024e8957 2024-11-23T14:27:31,822 DEBUG [StoreCloser-TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/b4be801b60634c5c8cb07bb0c54b3bc1 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/b4be801b60634c5c8cb07bb0c54b3bc1 2024-11-23T14:27:31,823 DEBUG [StoreCloser-TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/f072617d0f704b14b81cf2fe71ba8cd6 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/f072617d0f704b14b81cf2fe71ba8cd6 2024-11-23T14:27:31,824 DEBUG [StoreCloser-TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/916750a40ed6423789ec75c05b20021d, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/c22220c004c64c4aa36dbd9be665169f, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/72946c50ebc7494d90b6b42f7d7736d2, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/585e4a63cb474c81847372778bdf56b6, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/7e564de2c12a41468568aff8d64c2794, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/d7dc4004cbc046acb1af2a6c38e0367a, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/58cf4f7bb37f449588942376ef735ccd, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/09456c5655fc4cf999eec817cf5875a7, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/b552ac49af494df8ad9e95934a5a10f0, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/a3b3c332beb042889d19a073bd1dd8f2, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/8dbea164d23e4c8caa6213550b12bd6f, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/ebe62d163e254180a2a8f00b6474a384, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/e6ac4667ca624663b2fb01b84d429ba4, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/7db6442f16804dda8339807d9f5a92f2, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/bdb58ddf4477436596dcade7c22e96e2, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/b36a7b9c0c954713a2e39445deb63b8b, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/3b1f36063c384c4c972f5f894177fbea, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/ccd8920fbf5d47bc8a346e139b3c75e2, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/dedd24e335774be5a3ca6ab2398383ba, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/57e9ac60d9824733aa9341bf3c73c8a0, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/7dd5e0b989cc42a884e5f843fe93dbe1, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/921f51a9fb984010b7523b10db3906b1, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/d7113f3730854aa1a1896830369a8929] to archive 2024-11-23T14:27:31,825 DEBUG [StoreCloser-TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-23T14:27:31,826 DEBUG [StoreCloser-TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/916750a40ed6423789ec75c05b20021d to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/916750a40ed6423789ec75c05b20021d 2024-11-23T14:27:31,827 DEBUG [StoreCloser-TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/c22220c004c64c4aa36dbd9be665169f to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/c22220c004c64c4aa36dbd9be665169f 2024-11-23T14:27:31,828 DEBUG [StoreCloser-TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/72946c50ebc7494d90b6b42f7d7736d2 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/72946c50ebc7494d90b6b42f7d7736d2 2024-11-23T14:27:31,829 DEBUG [StoreCloser-TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/585e4a63cb474c81847372778bdf56b6 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/585e4a63cb474c81847372778bdf56b6 2024-11-23T14:27:31,830 DEBUG [StoreCloser-TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/7e564de2c12a41468568aff8d64c2794 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/7e564de2c12a41468568aff8d64c2794 2024-11-23T14:27:31,831 DEBUG [StoreCloser-TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/d7dc4004cbc046acb1af2a6c38e0367a to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/d7dc4004cbc046acb1af2a6c38e0367a 2024-11-23T14:27:31,831 DEBUG [StoreCloser-TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/58cf4f7bb37f449588942376ef735ccd to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/58cf4f7bb37f449588942376ef735ccd 2024-11-23T14:27:31,832 DEBUG [StoreCloser-TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/09456c5655fc4cf999eec817cf5875a7 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/09456c5655fc4cf999eec817cf5875a7 2024-11-23T14:27:31,833 DEBUG [StoreCloser-TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/b552ac49af494df8ad9e95934a5a10f0 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/b552ac49af494df8ad9e95934a5a10f0 2024-11-23T14:27:31,834 DEBUG [StoreCloser-TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/a3b3c332beb042889d19a073bd1dd8f2 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/a3b3c332beb042889d19a073bd1dd8f2 2024-11-23T14:27:31,834 DEBUG [StoreCloser-TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/8dbea164d23e4c8caa6213550b12bd6f to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/8dbea164d23e4c8caa6213550b12bd6f 2024-11-23T14:27:31,835 DEBUG [StoreCloser-TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/ebe62d163e254180a2a8f00b6474a384 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/ebe62d163e254180a2a8f00b6474a384 2024-11-23T14:27:31,836 DEBUG [StoreCloser-TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/e6ac4667ca624663b2fb01b84d429ba4 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/e6ac4667ca624663b2fb01b84d429ba4 2024-11-23T14:27:31,837 DEBUG [StoreCloser-TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/7db6442f16804dda8339807d9f5a92f2 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/7db6442f16804dda8339807d9f5a92f2 2024-11-23T14:27:31,838 DEBUG [StoreCloser-TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/bdb58ddf4477436596dcade7c22e96e2 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/bdb58ddf4477436596dcade7c22e96e2 2024-11-23T14:27:31,838 DEBUG [StoreCloser-TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/b36a7b9c0c954713a2e39445deb63b8b to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/b36a7b9c0c954713a2e39445deb63b8b 2024-11-23T14:27:31,839 DEBUG [StoreCloser-TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/3b1f36063c384c4c972f5f894177fbea to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/3b1f36063c384c4c972f5f894177fbea 2024-11-23T14:27:31,840 DEBUG [StoreCloser-TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/ccd8920fbf5d47bc8a346e139b3c75e2 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/ccd8920fbf5d47bc8a346e139b3c75e2 2024-11-23T14:27:31,840 DEBUG [StoreCloser-TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/dedd24e335774be5a3ca6ab2398383ba to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/dedd24e335774be5a3ca6ab2398383ba 2024-11-23T14:27:31,841 DEBUG [StoreCloser-TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/57e9ac60d9824733aa9341bf3c73c8a0 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/57e9ac60d9824733aa9341bf3c73c8a0 2024-11-23T14:27:31,842 DEBUG [StoreCloser-TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/7dd5e0b989cc42a884e5f843fe93dbe1 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/7dd5e0b989cc42a884e5f843fe93dbe1 2024-11-23T14:27:31,842 DEBUG [StoreCloser-TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/921f51a9fb984010b7523b10db3906b1 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/921f51a9fb984010b7523b10db3906b1 2024-11-23T14:27:31,843 DEBUG [StoreCloser-TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/d7113f3730854aa1a1896830369a8929 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/d7113f3730854aa1a1896830369a8929 2024-11-23T14:27:31,847 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/recovered.edits/403.seqid, newMaxSeqId=403, maxSeqId=1 2024-11-23T14:27:31,847 INFO [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. 2024-11-23T14:27:31,847 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1676): Region close journal for 3d5e64ff6f2c8eec1637e7e45ace6102: Waiting for close lock at 1732372049672Running coprocessor pre-close hooks at 1732372049672Disabling compacts and flushes for region at 1732372049672Disabling writes for close at 1732372050546 (+874 ms)Obtaining lock to block concurrent updates at 1732372050546Preparing flush snapshotting stores in 3d5e64ff6f2c8eec1637e7e45ace6102 at 1732372050546Finished memstore snapshotting TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1732372050546Flushing stores of TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102. at 1732372050546Flushing 3d5e64ff6f2c8eec1637e7e45ace6102/A: creating writer at 1732372050546Flushing 3d5e64ff6f2c8eec1637e7e45ace6102/A: appending metadata at 1732372050548 (+2 ms)Flushing 3d5e64ff6f2c8eec1637e7e45ace6102/A: closing flushed file at 1732372050549 (+1 ms)Flushing 3d5e64ff6f2c8eec1637e7e45ace6102/B: creating writer at 1732372050957 (+408 ms)Flushing 3d5e64ff6f2c8eec1637e7e45ace6102/B: appending metadata at 1732372050959 (+2 ms)Flushing 3d5e64ff6f2c8eec1637e7e45ace6102/B: closing flushed file at 1732372050959Flushing 3d5e64ff6f2c8eec1637e7e45ace6102/C: creating writer at 1732372051366 (+407 ms)Flushing 3d5e64ff6f2c8eec1637e7e45ace6102/C: appending metadata at 1732372051368 (+2 ms)Flushing 3d5e64ff6f2c8eec1637e7e45ace6102/C: closing flushed file at 1732372051369 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@385cc800: reopening flushed file at 1732372051775 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@44f507d0: reopening flushed file at 1732372051777 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@68db0c82: reopening flushed file at 1732372051781 (+4 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 3d5e64ff6f2c8eec1637e7e45ace6102 in 1239ms, sequenceid=400, compaction requested=true at 1732372051785 (+4 ms)Writing region close event to WAL at 1732372051844 (+59 ms)Running coprocessor post-close hooks at 1732372051847 (+3 ms)Closed at 1732372051847 2024-11-23T14:27:31,849 INFO [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] handler.UnassignRegionHandler(157): Closed 3d5e64ff6f2c8eec1637e7e45ace6102 2024-11-23T14:27:31,849 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=3d5e64ff6f2c8eec1637e7e45ace6102, regionState=CLOSED 2024-11-23T14:27:31,850 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=146, ppid=145, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 because future has completed 2024-11-23T14:27:31,852 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-11-23T14:27:31,852 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; CloseRegionProcedure 3d5e64ff6f2c8eec1637e7e45ace6102, server=c6f7b1897123,35385,1732371902767 in 2.3320 sec 2024-11-23T14:27:31,854 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-11-23T14:27:31,854 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3d5e64ff6f2c8eec1637e7e45ace6102, UNASSIGN in 2.3360 sec 2024-11-23T14:27:31,855 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-23T14:27:31,855 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.3400 sec 2024-11-23T14:27:31,856 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732372051856"}]},"ts":"1732372051856"} 2024-11-23T14:27:31,857 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-23T14:27:31,857 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-23T14:27:31,859 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.3550 sec 2024-11-23T14:27:32,052 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-23T14:27:33,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-23T14:27:33,671 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-23T14:27:33,671 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-23T14:27:33,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] procedure2.ProcedureExecutor(1139): Stored pid=147, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T14:27:33,673 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=147, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T14:27:33,673 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=147, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T14:27:33,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-23T14:27:33,675 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102 2024-11-23T14:27:33,677 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A, FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B, FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C, FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/recovered.edits] 2024-11-23T14:27:33,679 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/09b737907d7e42a9b14adbf0306549a2 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/09b737907d7e42a9b14adbf0306549a2 2024-11-23T14:27:33,680 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/0d882d1cf0744226a4c891976cdc4a0e to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/0d882d1cf0744226a4c891976cdc4a0e 2024-11-23T14:27:33,682 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/38ef3161ad5643629f36ea8d7ff5149e to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/38ef3161ad5643629f36ea8d7ff5149e 2024-11-23T14:27:33,683 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/d11b2a0645fe49d2bbe15695b9ce656b to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/d11b2a0645fe49d2bbe15695b9ce656b 2024-11-23T14:27:33,684 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/e87e0f7d0fe146e18f40397bf91dc1c1 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/A/e87e0f7d0fe146e18f40397bf91dc1c1 2024-11-23T14:27:33,687 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/1bd6d986a4d54eb1985f37cf6d68faef to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/1bd6d986a4d54eb1985f37cf6d68faef 2024-11-23T14:27:33,688 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/6acb234546204fe793f1208acf0ca4cd to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/6acb234546204fe793f1208acf0ca4cd 2024-11-23T14:27:33,689 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/772a18f207bd4da394e3f8a207a9dbf0 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/772a18f207bd4da394e3f8a207a9dbf0 2024-11-23T14:27:33,690 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/7a4785ba6c9641649fbcd1dcc0ee4d04 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/7a4785ba6c9641649fbcd1dcc0ee4d04 2024-11-23T14:27:33,691 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/dfa96b483d18484ab7402b7ebaf63e08 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/B/dfa96b483d18484ab7402b7ebaf63e08 2024-11-23T14:27:33,694 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/04f24fbd06cd49d58a5e1d5b9bdfa24c to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/04f24fbd06cd49d58a5e1d5b9bdfa24c 2024-11-23T14:27:33,695 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/44f0c5fdbb8f4ed7b9c462a49f086901 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/44f0c5fdbb8f4ed7b9c462a49f086901 2024-11-23T14:27:33,695 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/45d3cc28fb6a4df68b174fa1c2bb0956 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/45d3cc28fb6a4df68b174fa1c2bb0956 2024-11-23T14:27:33,696 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/4ba61b8127b44094a71cdab0da5f0117 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/4ba61b8127b44094a71cdab0da5f0117 2024-11-23T14:27:33,697 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/8f60fd452c564c7a9085649cfbaf8f55 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/C/8f60fd452c564c7a9085649cfbaf8f55 2024-11-23T14:27:33,701 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/recovered.edits/403.seqid to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102/recovered.edits/403.seqid 2024-11-23T14:27:33,702 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/3d5e64ff6f2c8eec1637e7e45ace6102 2024-11-23T14:27:33,702 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-23T14:27:33,704 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=147, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T14:27:33,706 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-23T14:27:33,708 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-23T14:27:33,709 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=147, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T14:27:33,709 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-23T14:27:33,709 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732372053709"}]},"ts":"9223372036854775807"} 2024-11-23T14:27:33,711 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-23T14:27:33,711 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 3d5e64ff6f2c8eec1637e7e45ace6102, NAME => 'TestAcidGuarantees,,1732372026079.3d5e64ff6f2c8eec1637e7e45ace6102.', STARTKEY => '', ENDKEY => ''}] 2024-11-23T14:27:33,711 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-23T14:27:33,712 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732372053711"}]},"ts":"9223372036854775807"} 2024-11-23T14:27:33,713 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-23T14:27:33,714 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=147, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T14:27:33,715 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 43 msec 2024-11-23T14:27:33,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-23T14:27:33,781 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-23T14:27:33,783 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-23T14:27:33,789 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=240 (was 240), OpenFileDescriptor=447 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=552 (was 630), ProcessCount=11 (was 12), AvailableMemoryMB=3102 (was 3179) 2024-11-23T14:27:33,797 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=240, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=552, ProcessCount=11, AvailableMemoryMB=3102 2024-11-23T14:27:33,798 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-23T14:27:33,799 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-23T14:27:33,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-23T14:27:33,801 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=148, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-23T14:27:33,801 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:33,801 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 148 2024-11-23T14:27:33,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-23T14:27:33,802 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=148, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-23T14:27:33,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742396_1572 (size=959) 2024-11-23T14:27:33,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-23T14:27:34,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-23T14:27:34,208 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a 2024-11-23T14:27:34,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742397_1573 (size=53) 2024-11-23T14:27:34,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-23T14:27:34,614 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T14:27:34,614 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 59fa5c31bb2297940cc7fa9d0ff623a3, disabling compactions & flushes 2024-11-23T14:27:34,614 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:34,614 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:34,614 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. after waiting 0 ms 2024-11-23T14:27:34,615 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:34,615 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:34,615 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 59fa5c31bb2297940cc7fa9d0ff623a3: Waiting for close lock at 1732372054614Disabling compacts and flushes for region at 1732372054614Disabling writes for close at 1732372054614Writing region close event to WAL at 1732372054615 (+1 ms)Closed at 1732372054615 2024-11-23T14:27:34,616 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=148, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-23T14:27:34,616 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732372054616"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732372054616"}]},"ts":"1732372054616"} 2024-11-23T14:27:34,617 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-23T14:27:34,618 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=148, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-23T14:27:34,618 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732372054618"}]},"ts":"1732372054618"} 2024-11-23T14:27:34,619 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-23T14:27:34,619 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=59fa5c31bb2297940cc7fa9d0ff623a3, ASSIGN}] 2024-11-23T14:27:34,620 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=59fa5c31bb2297940cc7fa9d0ff623a3, ASSIGN 2024-11-23T14:27:34,621 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=59fa5c31bb2297940cc7fa9d0ff623a3, ASSIGN; state=OFFLINE, location=c6f7b1897123,35385,1732371902767; forceNewPlan=false, retain=false 2024-11-23T14:27:34,771 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=59fa5c31bb2297940cc7fa9d0ff623a3, regionState=OPENING, regionLocation=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:34,773 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=59fa5c31bb2297940cc7fa9d0ff623a3, ASSIGN because future has completed 2024-11-23T14:27:34,773 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=150, ppid=149, state=RUNNABLE, hasLock=false; OpenRegionProcedure 59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767}] 2024-11-23T14:27:34,929 INFO [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:34,929 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(7752): Opening region: {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} 2024-11-23T14:27:34,929 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 59fa5c31bb2297940cc7fa9d0ff623a3 2024-11-23T14:27:34,929 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T14:27:34,930 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(7794): checking encryption for 59fa5c31bb2297940cc7fa9d0ff623a3 2024-11-23T14:27:34,930 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(7797): checking classloading for 59fa5c31bb2297940cc7fa9d0ff623a3 2024-11-23T14:27:34,931 INFO [StoreOpener-59fa5c31bb2297940cc7fa9d0ff623a3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 59fa5c31bb2297940cc7fa9d0ff623a3 2024-11-23T14:27:34,933 INFO [StoreOpener-59fa5c31bb2297940cc7fa9d0ff623a3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 59fa5c31bb2297940cc7fa9d0ff623a3 columnFamilyName A 2024-11-23T14:27:34,934 DEBUG [StoreOpener-59fa5c31bb2297940cc7fa9d0ff623a3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:34,934 INFO [StoreOpener-59fa5c31bb2297940cc7fa9d0ff623a3-1 {}] regionserver.HStore(327): Store=59fa5c31bb2297940cc7fa9d0ff623a3/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T14:27:34,935 INFO [StoreOpener-59fa5c31bb2297940cc7fa9d0ff623a3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 59fa5c31bb2297940cc7fa9d0ff623a3 2024-11-23T14:27:34,936 INFO [StoreOpener-59fa5c31bb2297940cc7fa9d0ff623a3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 59fa5c31bb2297940cc7fa9d0ff623a3 columnFamilyName B 2024-11-23T14:27:34,936 DEBUG [StoreOpener-59fa5c31bb2297940cc7fa9d0ff623a3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:34,937 INFO [StoreOpener-59fa5c31bb2297940cc7fa9d0ff623a3-1 {}] regionserver.HStore(327): Store=59fa5c31bb2297940cc7fa9d0ff623a3/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T14:27:34,937 INFO [StoreOpener-59fa5c31bb2297940cc7fa9d0ff623a3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 59fa5c31bb2297940cc7fa9d0ff623a3 2024-11-23T14:27:34,939 INFO [StoreOpener-59fa5c31bb2297940cc7fa9d0ff623a3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 59fa5c31bb2297940cc7fa9d0ff623a3 columnFamilyName C 2024-11-23T14:27:34,939 DEBUG [StoreOpener-59fa5c31bb2297940cc7fa9d0ff623a3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:34,939 INFO [StoreOpener-59fa5c31bb2297940cc7fa9d0ff623a3-1 {}] regionserver.HStore(327): Store=59fa5c31bb2297940cc7fa9d0ff623a3/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T14:27:34,939 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(1038): replaying wal for 59fa5c31bb2297940cc7fa9d0ff623a3 2024-11-23T14:27:34,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-23T14:27:34,940 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3 2024-11-23T14:27:34,941 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3 2024-11-23T14:27:34,942 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(1048): stopping wal replay for 59fa5c31bb2297940cc7fa9d0ff623a3 2024-11-23T14:27:34,942 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(1060): Cleaning up temporary data for 59fa5c31bb2297940cc7fa9d0ff623a3 2024-11-23T14:27:34,942 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-23T14:27:34,944 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(1093): writing seq id for 59fa5c31bb2297940cc7fa9d0ff623a3 2024-11-23T14:27:34,946 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T14:27:34,947 INFO [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(1114): Opened 59fa5c31bb2297940cc7fa9d0ff623a3; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72988558, jitterRate=0.0876142680644989}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-23T14:27:34,947 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 59fa5c31bb2297940cc7fa9d0ff623a3 2024-11-23T14:27:34,947 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(1006): Region open journal for 59fa5c31bb2297940cc7fa9d0ff623a3: Running coprocessor pre-open hook at 1732372054930Writing region info on filesystem at 1732372054930Initializing all the Stores at 1732372054931 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732372054931Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732372054931Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732372054931Cleaning up temporary data from old regions at 1732372054942 (+11 ms)Running coprocessor post-open hooks at 1732372054947 (+5 ms)Region opened successfully at 1732372054947 2024-11-23T14:27:34,948 INFO [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., pid=150, masterSystemTime=1732372054924 2024-11-23T14:27:34,950 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:34,950 INFO [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:34,950 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=59fa5c31bb2297940cc7fa9d0ff623a3, regionState=OPEN, openSeqNum=2, regionLocation=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:34,951 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=150, ppid=149, state=RUNNABLE, hasLock=false; OpenRegionProcedure 59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 because future has completed 2024-11-23T14:27:34,953 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=150, resume processing ppid=149 2024-11-23T14:27:34,953 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, ppid=149, state=SUCCESS, hasLock=false; OpenRegionProcedure 59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 in 179 msec 2024-11-23T14:27:34,954 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-23T14:27:34,954 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=59fa5c31bb2297940cc7fa9d0ff623a3, ASSIGN in 334 msec 2024-11-23T14:27:34,955 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=148, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-23T14:27:34,955 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732372054955"}]},"ts":"1732372054955"} 2024-11-23T14:27:34,956 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-23T14:27:34,957 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=148, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-23T14:27:34,958 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1580 sec 2024-11-23T14:27:35,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-23T14:27:35,950 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-23T14:27:35,951 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ae4ba4a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:27:35,951 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6f7b1897123,38947,-1 for getting cluster id 2024-11-23T14:27:35,952 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T14:27:35,952 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c88da961-458f-4375-b9d3-275f7f90019e' 2024-11-23T14:27:35,953 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T14:27:35,953 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c88da961-458f-4375-b9d3-275f7f90019e" 2024-11-23T14:27:35,953 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d8e031c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:27:35,953 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6f7b1897123,38947,-1] 2024-11-23T14:27:35,953 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T14:27:35,953 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:27:35,954 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56040, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T14:27:35,954 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e5104e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:27:35,955 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T14:27:35,955 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6f7b1897123,35385,1732371902767, seqNum=-1] 2024-11-23T14:27:35,955 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T14:27:35,956 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60428, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T14:27:35,957 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-23T14:27:35,958 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is c6f7b1897123,38947,1732371901705 2024-11-23T14:27:35,958 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@138805bb 2024-11-23T14:27:35,958 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-23T14:27:35,959 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56052, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-23T14:27:35,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-23T14:27:35,961 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-23T14:27:35,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] procedure2.ProcedureExecutor(1139): Stored pid=151, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-23T14:27:35,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742398_1574 (size=995) 2024-11-23T14:27:35,981 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-23T14:27:35,981 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-23T14:27:35,985 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=152, ppid=151, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-23T14:27:35,988 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=59fa5c31bb2297940cc7fa9d0ff623a3, REOPEN/MOVE}] 2024-11-23T14:27:35,989 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=153, ppid=152, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=59fa5c31bb2297940cc7fa9d0ff623a3, REOPEN/MOVE 2024-11-23T14:27:35,989 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=153 updating hbase:meta row=59fa5c31bb2297940cc7fa9d0ff623a3, regionState=CLOSING, regionLocation=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:35,992 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=153, ppid=152, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=59fa5c31bb2297940cc7fa9d0ff623a3, REOPEN/MOVE because future has completed 2024-11-23T14:27:35,992 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-23T14:27:35,992 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=154, ppid=153, state=RUNNABLE, hasLock=false; CloseRegionProcedure 59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767}] 2024-11-23T14:27:36,149 INFO [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] handler.UnassignRegionHandler(122): Close 59fa5c31bb2297940cc7fa9d0ff623a3 2024-11-23T14:27:36,149 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-23T14:27:36,149 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegion(1722): Closing 59fa5c31bb2297940cc7fa9d0ff623a3, disabling compactions & flushes 2024-11-23T14:27:36,149 INFO [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:36,149 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:36,149 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. after waiting 0 ms 2024-11-23T14:27:36,149 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:36,157 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-23T14:27:36,158 INFO [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:36,158 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegion(1676): Region close journal for 59fa5c31bb2297940cc7fa9d0ff623a3: Waiting for close lock at 1732372056149Running coprocessor pre-close hooks at 1732372056149Disabling compacts and flushes for region at 1732372056149Disabling writes for close at 1732372056149Writing region close event to WAL at 1732372056152 (+3 ms)Running coprocessor post-close hooks at 1732372056158 (+6 ms)Closed at 1732372056158 2024-11-23T14:27:36,158 WARN [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegionServer(3299): Not adding moved region record: 59fa5c31bb2297940cc7fa9d0ff623a3 to self. 2024-11-23T14:27:36,160 INFO [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] handler.UnassignRegionHandler(157): Closed 59fa5c31bb2297940cc7fa9d0ff623a3 2024-11-23T14:27:36,163 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=153 updating hbase:meta row=59fa5c31bb2297940cc7fa9d0ff623a3, regionState=CLOSED 2024-11-23T14:27:36,165 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=154, ppid=153, state=RUNNABLE, hasLock=false; CloseRegionProcedure 59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 because future has completed 2024-11-23T14:27:36,168 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=154, resume processing ppid=153 2024-11-23T14:27:36,168 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, ppid=153, state=SUCCESS, hasLock=false; CloseRegionProcedure 59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 in 174 msec 2024-11-23T14:27:36,169 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=153, ppid=152, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=59fa5c31bb2297940cc7fa9d0ff623a3, REOPEN/MOVE; state=CLOSED, location=c6f7b1897123,35385,1732371902767; forceNewPlan=false, retain=true 2024-11-23T14:27:36,320 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=153 updating hbase:meta row=59fa5c31bb2297940cc7fa9d0ff623a3, regionState=OPENING, regionLocation=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:36,322 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=153, ppid=152, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=59fa5c31bb2297940cc7fa9d0ff623a3, REOPEN/MOVE because future has completed 2024-11-23T14:27:36,322 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=153, state=RUNNABLE, hasLock=false; OpenRegionProcedure 59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767}] 2024-11-23T14:27:36,477 INFO [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:36,477 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(7752): Opening region: {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} 2024-11-23T14:27:36,478 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 59fa5c31bb2297940cc7fa9d0ff623a3 2024-11-23T14:27:36,478 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T14:27:36,478 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(7794): checking encryption for 59fa5c31bb2297940cc7fa9d0ff623a3 2024-11-23T14:27:36,478 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(7797): checking classloading for 59fa5c31bb2297940cc7fa9d0ff623a3 2024-11-23T14:27:36,479 INFO [StoreOpener-59fa5c31bb2297940cc7fa9d0ff623a3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 59fa5c31bb2297940cc7fa9d0ff623a3 2024-11-23T14:27:36,480 INFO [StoreOpener-59fa5c31bb2297940cc7fa9d0ff623a3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 59fa5c31bb2297940cc7fa9d0ff623a3 columnFamilyName A 2024-11-23T14:27:36,484 DEBUG [StoreOpener-59fa5c31bb2297940cc7fa9d0ff623a3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:36,484 INFO [StoreOpener-59fa5c31bb2297940cc7fa9d0ff623a3-1 {}] regionserver.HStore(327): Store=59fa5c31bb2297940cc7fa9d0ff623a3/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T14:27:36,486 INFO [StoreOpener-59fa5c31bb2297940cc7fa9d0ff623a3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 59fa5c31bb2297940cc7fa9d0ff623a3 2024-11-23T14:27:36,487 INFO [StoreOpener-59fa5c31bb2297940cc7fa9d0ff623a3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 59fa5c31bb2297940cc7fa9d0ff623a3 columnFamilyName B 2024-11-23T14:27:36,487 DEBUG [StoreOpener-59fa5c31bb2297940cc7fa9d0ff623a3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:36,488 INFO [StoreOpener-59fa5c31bb2297940cc7fa9d0ff623a3-1 {}] regionserver.HStore(327): Store=59fa5c31bb2297940cc7fa9d0ff623a3/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T14:27:36,488 INFO [StoreOpener-59fa5c31bb2297940cc7fa9d0ff623a3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 59fa5c31bb2297940cc7fa9d0ff623a3 2024-11-23T14:27:36,489 INFO [StoreOpener-59fa5c31bb2297940cc7fa9d0ff623a3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 59fa5c31bb2297940cc7fa9d0ff623a3 columnFamilyName C 2024-11-23T14:27:36,489 DEBUG [StoreOpener-59fa5c31bb2297940cc7fa9d0ff623a3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:36,489 INFO [StoreOpener-59fa5c31bb2297940cc7fa9d0ff623a3-1 {}] regionserver.HStore(327): Store=59fa5c31bb2297940cc7fa9d0ff623a3/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T14:27:36,489 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(1038): replaying wal for 59fa5c31bb2297940cc7fa9d0ff623a3 2024-11-23T14:27:36,490 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3 2024-11-23T14:27:36,491 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3 2024-11-23T14:27:36,492 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(1048): stopping wal replay for 59fa5c31bb2297940cc7fa9d0ff623a3 2024-11-23T14:27:36,492 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(1060): Cleaning up temporary data for 59fa5c31bb2297940cc7fa9d0ff623a3 2024-11-23T14:27:36,492 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-23T14:27:36,493 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(1093): writing seq id for 59fa5c31bb2297940cc7fa9d0ff623a3 2024-11-23T14:27:36,494 INFO [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(1114): Opened 59fa5c31bb2297940cc7fa9d0ff623a3; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68713815, jitterRate=0.02391563355922699}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-23T14:27:36,494 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 59fa5c31bb2297940cc7fa9d0ff623a3 2024-11-23T14:27:36,494 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(1006): Region open journal for 59fa5c31bb2297940cc7fa9d0ff623a3: Running coprocessor pre-open hook at 1732372056478Writing region info on filesystem at 1732372056478Initializing all the Stores at 1732372056478Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732372056479 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732372056479Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732372056479Cleaning up temporary data from old regions at 1732372056492 (+13 ms)Running coprocessor post-open hooks at 1732372056494 (+2 ms)Region opened successfully at 1732372056494 2024-11-23T14:27:36,495 INFO [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., pid=155, masterSystemTime=1732372056474 2024-11-23T14:27:36,497 DEBUG [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:36,497 INFO [RS_OPEN_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:36,498 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=153 updating hbase:meta row=59fa5c31bb2297940cc7fa9d0ff623a3, regionState=OPEN, openSeqNum=5, regionLocation=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:36,499 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=155, ppid=153, state=RUNNABLE, hasLock=false; OpenRegionProcedure 59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 because future has completed 2024-11-23T14:27:36,501 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=153 2024-11-23T14:27:36,501 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=153, state=SUCCESS, hasLock=false; OpenRegionProcedure 59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 in 178 msec 2024-11-23T14:27:36,503 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-23T14:27:36,503 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=59fa5c31bb2297940cc7fa9d0ff623a3, REOPEN/MOVE in 514 msec 2024-11-23T14:27:36,506 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=152, resume processing ppid=151 2024-11-23T14:27:36,506 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, ppid=151, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 519 msec 2024-11-23T14:27:36,509 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 546 msec 2024-11-23T14:27:36,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=151 2024-11-23T14:27:36,510 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-23T14:27:36,510 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@615c30, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:27:36,510 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6f7b1897123,38947,-1 for getting cluster id 2024-11-23T14:27:36,511 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T14:27:36,511 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c88da961-458f-4375-b9d3-275f7f90019e' 2024-11-23T14:27:36,512 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T14:27:36,512 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c88da961-458f-4375-b9d3-275f7f90019e" 2024-11-23T14:27:36,512 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6116542e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:27:36,512 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6f7b1897123,38947,-1] 2024-11-23T14:27:36,512 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T14:27:36,512 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:27:36,513 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56080, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T14:27:36,514 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ed4a711, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:27:36,515 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f0c6f93, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:27:36,515 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6f7b1897123,38947,-1 for getting cluster id 2024-11-23T14:27:36,515 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T14:27:36,516 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c88da961-458f-4375-b9d3-275f7f90019e' 2024-11-23T14:27:36,516 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T14:27:36,516 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c88da961-458f-4375-b9d3-275f7f90019e" 2024-11-23T14:27:36,516 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42a7e01c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:27:36,516 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6f7b1897123,38947,-1] 2024-11-23T14:27:36,517 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T14:27:36,517 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:27:36,518 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56098, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T14:27:36,520 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@632ec635, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:27:36,521 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d5ba127, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:27:36,521 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6f7b1897123,38947,-1 for getting cluster id 2024-11-23T14:27:36,521 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T14:27:36,522 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c88da961-458f-4375-b9d3-275f7f90019e' 2024-11-23T14:27:36,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T14:27:36,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c88da961-458f-4375-b9d3-275f7f90019e" 2024-11-23T14:27:36,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3861d923, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:27:36,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6f7b1897123,38947,-1] 2024-11-23T14:27:36,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T14:27:36,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:27:36,525 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56114, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T14:27:36,525 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ca4887a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:27:36,528 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@110f2d48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:27:36,528 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6f7b1897123,38947,-1 for getting cluster id 2024-11-23T14:27:36,528 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T14:27:36,531 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c88da961-458f-4375-b9d3-275f7f90019e' 2024-11-23T14:27:36,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T14:27:36,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c88da961-458f-4375-b9d3-275f7f90019e" 2024-11-23T14:27:36,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d64852f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:27:36,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6f7b1897123,38947,-1] 2024-11-23T14:27:36,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T14:27:36,533 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:27:36,533 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56142, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T14:27:36,534 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e58b61e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:27:36,535 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cbb80ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:27:36,535 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6f7b1897123,38947,-1 for getting cluster id 2024-11-23T14:27:36,535 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T14:27:36,535 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c88da961-458f-4375-b9d3-275f7f90019e' 2024-11-23T14:27:36,535 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T14:27:36,536 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c88da961-458f-4375-b9d3-275f7f90019e" 2024-11-23T14:27:36,536 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46291ee6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:27:36,536 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6f7b1897123,38947,-1] 2024-11-23T14:27:36,536 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T14:27:36,536 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:27:36,537 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56154, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T14:27:36,538 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22cffab3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:27:36,539 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66ae8872, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:27:36,539 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6f7b1897123,38947,-1 for getting cluster id 2024-11-23T14:27:36,539 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T14:27:36,540 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c88da961-458f-4375-b9d3-275f7f90019e' 2024-11-23T14:27:36,540 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T14:27:36,540 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c88da961-458f-4375-b9d3-275f7f90019e" 2024-11-23T14:27:36,540 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76740bfe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:27:36,540 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6f7b1897123,38947,-1] 2024-11-23T14:27:36,540 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T14:27:36,541 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:27:36,542 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56184, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T14:27:36,542 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43d1966, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:27:36,543 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3cf38cf8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:27:36,543 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6f7b1897123,38947,-1 for getting cluster id 2024-11-23T14:27:36,543 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T14:27:36,544 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c88da961-458f-4375-b9d3-275f7f90019e' 2024-11-23T14:27:36,544 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T14:27:36,544 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c88da961-458f-4375-b9d3-275f7f90019e" 2024-11-23T14:27:36,544 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@476aa6f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:27:36,544 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6f7b1897123,38947,-1] 2024-11-23T14:27:36,545 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T14:27:36,545 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:27:36,546 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56200, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T14:27:36,546 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5dcdb75b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:27:36,547 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55c3cfc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:27:36,547 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6f7b1897123,38947,-1 for getting cluster id 2024-11-23T14:27:36,547 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T14:27:36,548 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c88da961-458f-4375-b9d3-275f7f90019e' 2024-11-23T14:27:36,548 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T14:27:36,548 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c88da961-458f-4375-b9d3-275f7f90019e" 2024-11-23T14:27:36,549 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20b17961, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:27:36,549 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6f7b1897123,38947,-1] 2024-11-23T14:27:36,549 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T14:27:36,549 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:27:36,551 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56208, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T14:27:36,552 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2def56bd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:27:36,553 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58b067c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:27:36,553 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6f7b1897123,38947,-1 for getting cluster id 2024-11-23T14:27:36,553 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T14:27:36,554 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c88da961-458f-4375-b9d3-275f7f90019e' 2024-11-23T14:27:36,554 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T14:27:36,554 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c88da961-458f-4375-b9d3-275f7f90019e" 2024-11-23T14:27:36,554 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68d80a67, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:27:36,554 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6f7b1897123,38947,-1] 2024-11-23T14:27:36,555 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T14:27:36,555 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:27:36,556 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56228, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T14:27:36,556 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53421b75, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:27:36,557 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ebeeab1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:27:36,557 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6f7b1897123,38947,-1 for getting cluster id 2024-11-23T14:27:36,557 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T14:27:36,558 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c88da961-458f-4375-b9d3-275f7f90019e' 2024-11-23T14:27:36,558 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T14:27:36,558 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c88da961-458f-4375-b9d3-275f7f90019e" 2024-11-23T14:27:36,558 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@316fe3ca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:27:36,558 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6f7b1897123,38947,-1] 2024-11-23T14:27:36,558 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T14:27:36,558 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:27:36,559 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56236, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T14:27:36,559 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c2c605a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T14:27:36,560 DEBUG [Thread-2562 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T14:27:36,560 DEBUG [Thread-2547 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T14:27:36,560 DEBUG [Thread-2560 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T14:27:36,561 DEBUG [Thread-2556 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T14:27:36,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6f7b1897123,35385,1732371902767, seqNum=-1] 2024-11-23T14:27:36,561 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6f7b1897123,35385,1732371902767, seqNum=-1] 2024-11-23T14:27:36,561 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T14:27:36,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T14:27:36,561 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6f7b1897123,35385,1732371902767, seqNum=-1] 2024-11-23T14:27:36,561 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T14:27:36,562 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60434, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T14:27:36,563 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60432, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T14:27:36,564 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6f7b1897123,35385,1732371902767, seqNum=-1] 2024-11-23T14:27:36,564 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60448, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T14:27:36,564 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T14:27:36,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5] 2024-11-23T14:27:36,565 DEBUG [Thread-2545 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T14:27:36,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5] 2024-11-23T14:27:36,566 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6f7b1897123,35385,1732371902767, seqNum=-1] 2024-11-23T14:27:36,564 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5] 2024-11-23T14:27:36,566 DEBUG [Thread-2549 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T14:27:36,566 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T14:27:36,567 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60462, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T14:27:36,567 DEBUG [Thread-2558 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T14:27:36,567 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60468, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T14:27:36,568 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-23T14:27:36,568 DEBUG [Thread-2543 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T14:27:36,569 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6f7b1897123,35385,1732371902767, seqNum=-1] 2024-11-23T14:27:36,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-23T14:27:36,569 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T14:27:36,569 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5] 2024-11-23T14:27:36,569 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6f7b1897123,35385,1732371902767, seqNum=-1] 2024-11-23T14:27:36,569 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T14:27:36,570 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5] 2024-11-23T14:27:36,571 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60472, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T14:27:36,571 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T14:27:36,571 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60470, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T14:27:36,571 DEBUG [Thread-2551 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T14:27:36,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-23T14:27:36,572 DEBUG [Thread-2549 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6f7b1897123,35385,1732371902767, seqNum=-1] 2024-11-23T14:27:36,573 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6f7b1897123,35385,1732371902767, seqNum=-1] 2024-11-23T14:27:36,573 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T14:27:36,573 DEBUG [Thread-2549 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T14:27:36,574 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5] 2024-11-23T14:27:36,574 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T14:27:36,574 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60476, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T14:27:36,574 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T14:27:36,575 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60492, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T14:27:36,576 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5] 2024-11-23T14:27:36,581 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5] 2024-11-23T14:27:36,581 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5] 2024-11-23T14:27:36,584 DEBUG [Thread-2554 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T14:27:36,586 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6f7b1897123,35385,1732371902767, seqNum=-1] 2024-11-23T14:27:36,586 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T14:27:36,589 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T14:27:36,589 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 59fa5c31bb2297940cc7fa9d0ff623a3 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-23T14:27:36,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on 59fa5c31bb2297940cc7fa9d0ff623a3 2024-11-23T14:27:36,591 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60498, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T14:27:36,594 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5] 2024-11-23T14:27:36,636 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123d840820bf46641958a2dce555164532b_59fa5c31bb2297940cc7fa9d0ff623a3 is 50, key is test_row_0/A:col10/1732372056585/Put/seqid=0 2024-11-23T14:27:36,645 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:36,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60470 deadline: 1732372116635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:36,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742399_1575 (size=9714) 2024-11-23T14:27:36,645 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:36,646 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:36,646 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:36,646 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:36,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60492 deadline: 1732372116635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:36,646 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:36,646 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:36,646 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:36,646 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:36,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:36,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60448 deadline: 1732372116641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:36,647 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:36,647 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:36,647 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:36,647 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:36,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60476 deadline: 1732372116636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:36,648 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:36,648 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:36,648 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:36,653 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:36,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60468 deadline: 1732372116646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:36,654 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:36,654 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:36,654 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:36,658 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123d840820bf46641958a2dce555164532b_59fa5c31bb2297940cc7fa9d0ff623a3 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123d840820bf46641958a2dce555164532b_59fa5c31bb2297940cc7fa9d0ff623a3 2024-11-23T14:27:36,659 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/A/f96f960d09894b3fb8c4032161828808, store: [table=TestAcidGuarantees family=A region=59fa5c31bb2297940cc7fa9d0ff623a3] 2024-11-23T14:27:36,660 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/A/f96f960d09894b3fb8c4032161828808 is 175, key is test_row_0/A:col10/1732372056585/Put/seqid=0 2024-11-23T14:27:36,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742400_1576 (size=22361) 2024-11-23T14:27:36,678 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=17, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/A/f96f960d09894b3fb8c4032161828808 2024-11-23T14:27:36,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-23T14:27:36,712 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/B/0d45164106f64a3991d287fe0c8058e9 is 50, key is test_row_0/B:col10/1732372056585/Put/seqid=0 2024-11-23T14:27:36,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742401_1577 (size=9657) 2024-11-23T14:27:36,719 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/B/0d45164106f64a3991d287fe0c8058e9 2024-11-23T14:27:36,735 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-23T14:27:36,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:36,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. as already flushing 2024-11-23T14:27:36,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:36,735 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:36,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:36,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:36,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:36,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60470 deadline: 1732372116750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:36,752 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:36,752 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:36,752 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:36,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:36,752 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:36,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60492 deadline: 1732372116751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:36,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60448 deadline: 1732372116751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:36,753 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:36,753 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:36,753 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:36,753 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:36,753 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:36,753 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:36,753 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:36,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60476 deadline: 1732372116751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:36,754 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:36,754 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:36,754 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:36,760 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:36,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60468 deadline: 1732372116760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:36,761 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:36,761 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:36,761 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:36,763 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/C/8260e3a92bb0437e8b63f09846d93390 is 50, key is test_row_0/C:col10/1732372056585/Put/seqid=0 2024-11-23T14:27:36,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742402_1578 (size=9657) 2024-11-23T14:27:36,889 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-23T14:27:36,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:36,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. as already flushing 2024-11-23T14:27:36,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:36,889 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:36,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:36,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-23T14:27:36,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:36,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:36,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:36,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60448 deadline: 1732372116960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:36,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60492 deadline: 1732372116960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:36,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:36,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:36,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:36,962 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:36,962 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:36,962 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:36,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:36,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60476 deadline: 1732372116961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:36,963 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:36,963 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:36,963 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:36,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:36,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60470 deadline: 1732372116960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:36,965 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:36,965 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:36,965 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:36,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:36,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60468 deadline: 1732372116970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:36,972 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:36,972 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:36,972 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:37,052 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-23T14:27:37,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:37,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. as already flushing 2024-11-23T14:27:37,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:37,052 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:37,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:37,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:37,182 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/C/8260e3a92bb0437e8b63f09846d93390 2024-11-23T14:27:37,187 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/A/f96f960d09894b3fb8c4032161828808 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/f96f960d09894b3fb8c4032161828808 2024-11-23T14:27:37,192 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/f96f960d09894b3fb8c4032161828808, entries=100, sequenceid=17, filesize=21.8 K 2024-11-23T14:27:37,193 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/B/0d45164106f64a3991d287fe0c8058e9 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/0d45164106f64a3991d287fe0c8058e9 2024-11-23T14:27:37,200 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/0d45164106f64a3991d287fe0c8058e9, entries=100, sequenceid=17, filesize=9.4 K 2024-11-23T14:27:37,201 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/C/8260e3a92bb0437e8b63f09846d93390 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/8260e3a92bb0437e8b63f09846d93390 2024-11-23T14:27:37,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-23T14:27:37,206 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/8260e3a92bb0437e8b63f09846d93390, entries=100, sequenceid=17, filesize=9.4 K 2024-11-23T14:27:37,208 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 59fa5c31bb2297940cc7fa9d0ff623a3 in 619ms, sequenceid=17, compaction requested=false 2024-11-23T14:27:37,208 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 59fa5c31bb2297940cc7fa9d0ff623a3: 2024-11-23T14:27:37,211 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-23T14:27:37,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:37,211 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing 59fa5c31bb2297940cc7fa9d0ff623a3 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-23T14:27:37,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123261bdf9809c04d008a99f50a112f3173_59fa5c31bb2297940cc7fa9d0ff623a3 is 50, key is test_row_0/A:col10/1732372056631/Put/seqid=0 2024-11-23T14:27:37,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742403_1579 (size=12154) 2024-11-23T14:27:37,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:37,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on 59fa5c31bb2297940cc7fa9d0ff623a3 2024-11-23T14:27:37,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. as already flushing 2024-11-23T14:27:37,277 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123261bdf9809c04d008a99f50a112f3173_59fa5c31bb2297940cc7fa9d0ff623a3 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123261bdf9809c04d008a99f50a112f3173_59fa5c31bb2297940cc7fa9d0ff623a3 2024-11-23T14:27:37,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/A/8e5c4798f8fd463495bc04d01ac121f9, store: [table=TestAcidGuarantees family=A region=59fa5c31bb2297940cc7fa9d0ff623a3] 2024-11-23T14:27:37,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/A/8e5c4798f8fd463495bc04d01ac121f9 is 175, key is test_row_0/A:col10/1732372056631/Put/seqid=0 2024-11-23T14:27:37,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:37,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60492 deadline: 1732372117282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:37,291 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:37,291 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:37,291 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:37,293 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:37,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60476 deadline: 1732372117288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:37,294 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:37,294 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:37,294 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:37,294 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:37,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60448 deadline: 1732372117289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:37,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:37,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:37,295 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:37,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:37,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60468 deadline: 1732372117290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:37,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:37,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60470 deadline: 1732372117290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:37,296 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:37,296 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:37,296 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:37,296 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:37,296 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:37,297 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:37,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742404_1580 (size=30955) 2024-11-23T14:27:37,329 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=41, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/A/8e5c4798f8fd463495bc04d01ac121f9 2024-11-23T14:27:37,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/B/08b1fa65873c41528b538b75599d649f is 50, key is test_row_0/B:col10/1732372056631/Put/seqid=0 2024-11-23T14:27:37,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742405_1581 (size=12001) 2024-11-23T14:27:37,378 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/B/08b1fa65873c41528b538b75599d649f 2024-11-23T14:27:37,391 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-23T14:27:37,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:37,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60468 deadline: 1732372117401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:37,405 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:37,405 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:37,405 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:37,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:37,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60492 deadline: 1732372117403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:37,409 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:37,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:37,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60476 deadline: 1732372117403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:37,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60448 deadline: 1732372117403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:37,409 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:37,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:37,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:37,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:37,409 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:37,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:37,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60470 deadline: 1732372117403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:37,409 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:37,409 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:37,409 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:37,409 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:37,409 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:37,409 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:37,409 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:37,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/C/ca9cb7616cd54e62af8f8c1c7ff85b7b is 50, key is test_row_0/C:col10/1732372056631/Put/seqid=0 2024-11-23T14:27:37,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742406_1582 (size=12001) 2024-11-23T14:27:37,611 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:37,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60468 deadline: 1732372117610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:37,612 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:37,612 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:37,612 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:37,621 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:37,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60476 deadline: 1732372117621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:37,621 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:37,621 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:37,621 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:37,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60470 deadline: 1732372117621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:37,621 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:37,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60448 deadline: 1732372117621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:37,621 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:37,622 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:37,622 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:37,622 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:37,622 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:37,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60492 deadline: 1732372117621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:37,622 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:37,622 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:37,622 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:37,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:37,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:37,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:37,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-23T14:27:37,852 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/C/ca9cb7616cd54e62af8f8c1c7ff85b7b 2024-11-23T14:27:37,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/A/8e5c4798f8fd463495bc04d01ac121f9 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/8e5c4798f8fd463495bc04d01ac121f9 2024-11-23T14:27:37,861 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/8e5c4798f8fd463495bc04d01ac121f9, entries=150, sequenceid=41, filesize=30.2 K 2024-11-23T14:27:37,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/B/08b1fa65873c41528b538b75599d649f as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/08b1fa65873c41528b538b75599d649f 2024-11-23T14:27:37,868 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/08b1fa65873c41528b538b75599d649f, entries=150, sequenceid=41, filesize=11.7 K 2024-11-23T14:27:37,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/C/ca9cb7616cd54e62af8f8c1c7ff85b7b as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/ca9cb7616cd54e62af8f8c1c7ff85b7b 2024-11-23T14:27:37,879 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/ca9cb7616cd54e62af8f8c1c7ff85b7b, entries=150, sequenceid=41, filesize=11.7 K 2024-11-23T14:27:37,880 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for 59fa5c31bb2297940cc7fa9d0ff623a3 in 669ms, sequenceid=41, compaction requested=false 2024-11-23T14:27:37,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for 59fa5c31bb2297940cc7fa9d0ff623a3: 2024-11-23T14:27:37,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:37,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-23T14:27:37,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-23T14:27:37,888 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-23T14:27:37,888 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3120 sec 2024-11-23T14:27:37,892 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 1.3220 sec 2024-11-23T14:27:37,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on 59fa5c31bb2297940cc7fa9d0ff623a3 2024-11-23T14:27:37,925 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T14:27:37,926 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 59fa5c31bb2297940cc7fa9d0ff623a3 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-23T14:27:37,946 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123f6fa48ea4fc043c39f68ef50dce7372e_59fa5c31bb2297940cc7fa9d0ff623a3 is 50, key is test_row_0/A:col10/1732372057288/Put/seqid=0 2024-11-23T14:27:37,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:37,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60492 deadline: 1732372117977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:37,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:37,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60468 deadline: 1732372117977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:37,983 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:37,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:37,983 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:37,983 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:37,983 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:37,983 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:37,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60476 deadline: 1732372117977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:37,983 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:37,983 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:37,983 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:37,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:37,983 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:37,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60448 deadline: 1732372117979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:37,984 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:37,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60470 deadline: 1732372117979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:37,984 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:37,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:37,984 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:37,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:37,984 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:37,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:37,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742407_1583 (size=12154) 2024-11-23T14:27:38,092 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:38,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60468 deadline: 1732372118090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:38,092 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:38,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60492 deadline: 1732372118090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:38,092 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:38,092 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:38,092 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:38,092 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:38,092 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:38,092 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:38,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60470 deadline: 1732372118090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:38,092 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:38,093 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:38,093 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:38,093 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:38,093 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:38,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60476 deadline: 1732372118090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:38,093 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:38,093 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:38,093 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:38,093 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:38,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60448 deadline: 1732372118090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:38,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:38,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:38,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:38,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:38,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60492 deadline: 1732372118300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:38,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:38,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60476 deadline: 1732372118301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:38,303 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:38,303 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:38,303 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:38,303 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:38,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:38,303 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:38,303 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:38,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60448 deadline: 1732372118301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:38,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:38,304 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:38,304 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:38,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60470 deadline: 1732372118301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:38,304 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:38,304 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:38,304 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:38,304 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:38,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60468 deadline: 1732372118301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:38,304 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:38,304 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:38,304 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:38,304 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:38,385 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:38,389 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123f6fa48ea4fc043c39f68ef50dce7372e_59fa5c31bb2297940cc7fa9d0ff623a3 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123f6fa48ea4fc043c39f68ef50dce7372e_59fa5c31bb2297940cc7fa9d0ff623a3 2024-11-23T14:27:38,390 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/A/b133da13ac6b49538a06d643f6e43d41, store: [table=TestAcidGuarantees family=A region=59fa5c31bb2297940cc7fa9d0ff623a3] 2024-11-23T14:27:38,391 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/A/b133da13ac6b49538a06d643f6e43d41 is 175, key is test_row_0/A:col10/1732372057288/Put/seqid=0 2024-11-23T14:27:38,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742408_1584 (size=30955) 2024-11-23T14:27:38,395 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=55, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/A/b133da13ac6b49538a06d643f6e43d41 2024-11-23T14:27:38,421 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/B/3056c23946c14b5794da0a1a93560a1a is 50, key is test_row_0/B:col10/1732372057288/Put/seqid=0 2024-11-23T14:27:38,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742409_1585 (size=12001) 2024-11-23T14:27:38,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:38,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60492 deadline: 1732372118612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:38,613 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:38,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:38,613 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:38,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60476 deadline: 1732372118612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:38,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60448 deadline: 1732372118613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:38,613 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:38,613 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:38,613 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:38,614 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:38,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:38,614 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:38,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60470 deadline: 1732372118613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:38,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:38,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:38,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:38,614 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:38,614 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:38,614 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:38,616 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:38,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60468 deadline: 1732372118613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:38,616 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:38,616 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:38,617 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:38,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-23T14:27:38,720 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T14:27:38,720 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-23T14:27:38,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-23T14:27:38,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-23T14:27:38,725 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T14:27:38,726 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T14:27:38,726 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T14:27:38,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-23T14:27:38,846 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/B/3056c23946c14b5794da0a1a93560a1a 2024-11-23T14:27:38,853 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/C/fa89e087536c40069d544160d5d019f0 is 50, key is test_row_0/C:col10/1732372057288/Put/seqid=0 2024-11-23T14:27:38,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742410_1586 (size=12001) 2024-11-23T14:27:38,879 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-23T14:27:38,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:38,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. as already flushing 2024-11-23T14:27:38,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:38,879 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:38,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:38,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:39,032 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-23T14:27:39,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:39,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. as already flushing 2024-11-23T14:27:39,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:39,032 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:39,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:39,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:39,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-23T14:27:39,122 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:39,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60492 deadline: 1732372119121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:39,122 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:39,122 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:39,122 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:39,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60470 deadline: 1732372119121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:39,122 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:39,122 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:39,123 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:39,123 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:39,123 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:39,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60468 deadline: 1732372119121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:39,123 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:39,123 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:39,123 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:39,123 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:39,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60448 deadline: 1732372119121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:39,123 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:39,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60476 deadline: 1732372119121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:39,123 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:39,123 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:39,123 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:39,124 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:39,124 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:39,124 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:39,193 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-23T14:27:39,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:39,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. as already flushing 2024-11-23T14:27:39,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:39,193 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:39,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:39,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:39,266 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/C/fa89e087536c40069d544160d5d019f0 2024-11-23T14:27:39,271 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/A/b133da13ac6b49538a06d643f6e43d41 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/b133da13ac6b49538a06d643f6e43d41 2024-11-23T14:27:39,275 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/b133da13ac6b49538a06d643f6e43d41, entries=150, sequenceid=55, filesize=30.2 K 2024-11-23T14:27:39,276 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/B/3056c23946c14b5794da0a1a93560a1a as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/3056c23946c14b5794da0a1a93560a1a 2024-11-23T14:27:39,279 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/3056c23946c14b5794da0a1a93560a1a, entries=150, sequenceid=55, filesize=11.7 K 2024-11-23T14:27:39,279 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/C/fa89e087536c40069d544160d5d019f0 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/fa89e087536c40069d544160d5d019f0 2024-11-23T14:27:39,282 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/fa89e087536c40069d544160d5d019f0, entries=150, sequenceid=55, filesize=11.7 K 2024-11-23T14:27:39,283 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for 59fa5c31bb2297940cc7fa9d0ff623a3 in 1357ms, sequenceid=55, compaction requested=true 2024-11-23T14:27:39,283 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 59fa5c31bb2297940cc7fa9d0ff623a3: 2024-11-23T14:27:39,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 59fa5c31bb2297940cc7fa9d0ff623a3:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T14:27:39,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:27:39,283 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:27:39,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 59fa5c31bb2297940cc7fa9d0ff623a3:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T14:27:39,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:27:39,283 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:27:39,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 59fa5c31bb2297940cc7fa9d0ff623a3:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T14:27:39,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:27:39,283 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 84271 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:27:39,283 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1541): 59fa5c31bb2297940cc7fa9d0ff623a3/A is initiating minor compaction (all files) 2024-11-23T14:27:39,284 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 59fa5c31bb2297940cc7fa9d0ff623a3/A in TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:39,284 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/f96f960d09894b3fb8c4032161828808, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/8e5c4798f8fd463495bc04d01ac121f9, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/b133da13ac6b49538a06d643f6e43d41] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp, totalSize=82.3 K 2024-11-23T14:27:39,284 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:39,284 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. files: [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/f96f960d09894b3fb8c4032161828808, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/8e5c4798f8fd463495bc04d01ac121f9, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/b133da13ac6b49538a06d643f6e43d41] 2024-11-23T14:27:39,284 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:27:39,284 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting f96f960d09894b3fb8c4032161828808, keycount=100, bloomtype=ROW, size=21.8 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1732372056585 2024-11-23T14:27:39,284 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1541): 59fa5c31bb2297940cc7fa9d0ff623a3/B is initiating minor compaction (all files) 2024-11-23T14:27:39,284 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 59fa5c31bb2297940cc7fa9d0ff623a3/B in TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:39,285 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/0d45164106f64a3991d287fe0c8058e9, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/08b1fa65873c41528b538b75599d649f, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/3056c23946c14b5794da0a1a93560a1a] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp, totalSize=32.9 K 2024-11-23T14:27:39,285 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8e5c4798f8fd463495bc04d01ac121f9, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1732372056631 2024-11-23T14:27:39,285 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d45164106f64a3991d287fe0c8058e9, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1732372056585 2024-11-23T14:27:39,285 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting b133da13ac6b49538a06d643f6e43d41, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1732372057286 2024-11-23T14:27:39,285 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 08b1fa65873c41528b538b75599d649f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1732372056631 2024-11-23T14:27:39,285 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 3056c23946c14b5794da0a1a93560a1a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1732372057286 2024-11-23T14:27:39,290 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=59fa5c31bb2297940cc7fa9d0ff623a3] 2024-11-23T14:27:39,292 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 59fa5c31bb2297940cc7fa9d0ff623a3#B#compaction#501 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:27:39,292 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/B/ebb73acb95214f769491fbfccb36903f is 50, key is test_row_0/B:col10/1732372057288/Put/seqid=0 2024-11-23T14:27:39,298 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241123a858fdbc51874ebd894f601b0153ecfe_59fa5c31bb2297940cc7fa9d0ff623a3 store=[table=TestAcidGuarantees family=A region=59fa5c31bb2297940cc7fa9d0ff623a3] 2024-11-23T14:27:39,299 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241123a858fdbc51874ebd894f601b0153ecfe_59fa5c31bb2297940cc7fa9d0ff623a3, store=[table=TestAcidGuarantees family=A region=59fa5c31bb2297940cc7fa9d0ff623a3] 2024-11-23T14:27:39,299 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123a858fdbc51874ebd894f601b0153ecfe_59fa5c31bb2297940cc7fa9d0ff623a3 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=59fa5c31bb2297940cc7fa9d0ff623a3] 2024-11-23T14:27:39,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742411_1587 (size=12104) 2024-11-23T14:27:39,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-23T14:27:39,350 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-23T14:27:39,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:39,350 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing 59fa5c31bb2297940cc7fa9d0ff623a3 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-23T14:27:39,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742412_1588 (size=4469) 2024-11-23T14:27:39,360 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 59fa5c31bb2297940cc7fa9d0ff623a3#A#compaction#500 average throughput is 0.35 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:27:39,361 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/A/b4c583d41d4b47938ca835dd1bd635d6 is 175, key is test_row_0/A:col10/1732372057288/Put/seqid=0 2024-11-23T14:27:39,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411235abff9c1f6e149f2813b81115416a8b4_59fa5c31bb2297940cc7fa9d0ff623a3 is 50, key is test_row_0/A:col10/1732372057973/Put/seqid=0 2024-11-23T14:27:39,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742413_1589 (size=31058) 2024-11-23T14:27:39,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742414_1590 (size=12154) 2024-11-23T14:27:39,729 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/B/ebb73acb95214f769491fbfccb36903f as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/ebb73acb95214f769491fbfccb36903f 2024-11-23T14:27:39,733 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 59fa5c31bb2297940cc7fa9d0ff623a3/B of 59fa5c31bb2297940cc7fa9d0ff623a3 into ebb73acb95214f769491fbfccb36903f(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:27:39,733 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 59fa5c31bb2297940cc7fa9d0ff623a3: 2024-11-23T14:27:39,733 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., storeName=59fa5c31bb2297940cc7fa9d0ff623a3/B, priority=13, startTime=1732372059283; duration=0sec 2024-11-23T14:27:39,733 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:27:39,733 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 59fa5c31bb2297940cc7fa9d0ff623a3:B 2024-11-23T14:27:39,734 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:27:39,734 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:27:39,734 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1541): 59fa5c31bb2297940cc7fa9d0ff623a3/C is initiating minor compaction (all files) 2024-11-23T14:27:39,735 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 59fa5c31bb2297940cc7fa9d0ff623a3/C in TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:39,735 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/8260e3a92bb0437e8b63f09846d93390, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/ca9cb7616cd54e62af8f8c1c7ff85b7b, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/fa89e087536c40069d544160d5d019f0] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp, totalSize=32.9 K 2024-11-23T14:27:39,735 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 8260e3a92bb0437e8b63f09846d93390, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1732372056585 2024-11-23T14:27:39,735 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting ca9cb7616cd54e62af8f8c1c7ff85b7b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1732372056631 2024-11-23T14:27:39,735 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting fa89e087536c40069d544160d5d019f0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1732372057286 2024-11-23T14:27:39,743 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 59fa5c31bb2297940cc7fa9d0ff623a3#C#compaction#503 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:27:39,744 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/C/4ebe0ca8dc9b494c9730827905c76429 is 50, key is test_row_0/C:col10/1732372057288/Put/seqid=0 2024-11-23T14:27:39,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742415_1591 (size=12104) 2024-11-23T14:27:39,777 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/A/b4c583d41d4b47938ca835dd1bd635d6 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/b4c583d41d4b47938ca835dd1bd635d6 2024-11-23T14:27:39,783 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 59fa5c31bb2297940cc7fa9d0ff623a3/A of 59fa5c31bb2297940cc7fa9d0ff623a3 into b4c583d41d4b47938ca835dd1bd635d6(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:27:39,783 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 59fa5c31bb2297940cc7fa9d0ff623a3: 2024-11-23T14:27:39,783 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., storeName=59fa5c31bb2297940cc7fa9d0ff623a3/A, priority=13, startTime=1732372059283; duration=0sec 2024-11-23T14:27:39,783 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:27:39,783 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 59fa5c31bb2297940cc7fa9d0ff623a3:A 2024-11-23T14:27:39,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,790 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411235abff9c1f6e149f2813b81115416a8b4_59fa5c31bb2297940cc7fa9d0ff623a3 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411235abff9c1f6e149f2813b81115416a8b4_59fa5c31bb2297940cc7fa9d0ff623a3 2024-11-23T14:27:39,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/A/b28956adc03f49fe85d832ee99a4ff80, store: [table=TestAcidGuarantees family=A region=59fa5c31bb2297940cc7fa9d0ff623a3] 2024-11-23T14:27:39,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/A/b28956adc03f49fe85d832ee99a4ff80 is 175, key is test_row_0/A:col10/1732372057973/Put/seqid=0 2024-11-23T14:27:39,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742416_1592 (size=30955) 2024-11-23T14:27:39,805 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=78, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/A/b28956adc03f49fe85d832ee99a4ff80 2024-11-23T14:27:39,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/B/dcfa774201a24337986d60ee24efdde8 is 50, key is test_row_0/B:col10/1732372057973/Put/seqid=0 2024-11-23T14:27:39,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742417_1593 (size=12001) 2024-11-23T14:27:39,832 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/B/dcfa774201a24337986d60ee24efdde8 2024-11-23T14:27:39,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/C/8664b2065ba14f07b640dd8aa9bd854a is 50, key is test_row_0/C:col10/1732372057973/Put/seqid=0 2024-11-23T14:27:39,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742418_1594 (size=12001) 2024-11-23T14:27:39,856 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/C/8664b2065ba14f07b640dd8aa9bd854a 2024-11-23T14:27:39,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-23T14:27:39,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/A/b28956adc03f49fe85d832ee99a4ff80 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/b28956adc03f49fe85d832ee99a4ff80 2024-11-23T14:27:39,879 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/b28956adc03f49fe85d832ee99a4ff80, entries=150, sequenceid=78, filesize=30.2 K 2024-11-23T14:27:39,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/B/dcfa774201a24337986d60ee24efdde8 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/dcfa774201a24337986d60ee24efdde8 2024-11-23T14:27:39,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,886 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/dcfa774201a24337986d60ee24efdde8, entries=150, sequenceid=78, filesize=11.7 K 2024-11-23T14:27:39,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/C/8664b2065ba14f07b640dd8aa9bd854a as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/8664b2065ba14f07b640dd8aa9bd854a 2024-11-23T14:27:39,890 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/8664b2065ba14f07b640dd8aa9bd854a, entries=150, sequenceid=78, filesize=11.7 K 2024-11-23T14:27:39,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,891 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=0 B/0 for 59fa5c31bb2297940cc7fa9d0ff623a3 in 541ms, sequenceid=78, compaction requested=false 2024-11-23T14:27:39,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for 59fa5c31bb2297940cc7fa9d0ff623a3: 2024-11-23T14:27:39,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:39,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-23T14:27:39,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-23T14:27:39,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,900 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-23T14:27:39,901 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1670 sec 2024-11-23T14:27:39,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,903 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 1.1810 sec 2024-11-23T14:27:39,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:39,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,161 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/C/4ebe0ca8dc9b494c9730827905c76429 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/4ebe0ca8dc9b494c9730827905c76429 2024-11-23T14:27:40,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,175 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 59fa5c31bb2297940cc7fa9d0ff623a3/C of 59fa5c31bb2297940cc7fa9d0ff623a3 into 4ebe0ca8dc9b494c9730827905c76429(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:27:40,175 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 59fa5c31bb2297940cc7fa9d0ff623a3: 2024-11-23T14:27:40,175 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., storeName=59fa5c31bb2297940cc7fa9d0ff623a3/C, priority=13, startTime=1732372059283; duration=0sec 2024-11-23T14:27:40,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,175 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:27:40,175 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 59fa5c31bb2297940cc7fa9d0ff623a3:C 2024-11-23T14:27:40,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on 59fa5c31bb2297940cc7fa9d0ff623a3 2024-11-23T14:27:40,197 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T14:27:40,197 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 59fa5c31bb2297940cc7fa9d0ff623a3 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-23T14:27:40,206 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123e6afb821ebe64211abbde642cd16eb77_59fa5c31bb2297940cc7fa9d0ff623a3 is 50, key is test_row_0/A:col10/1732372060191/Put/seqid=0 2024-11-23T14:27:40,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742419_1595 (size=12154) 2024-11-23T14:27:40,239 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:40,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60492 deadline: 1732372120233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:40,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:40,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60470 deadline: 1732372120233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:40,240 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:40,240 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:40,240 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:40,240 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:40,240 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:40,240 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:40,241 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:40,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60468 deadline: 1732372120236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:40,241 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:40,241 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:40,241 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:40,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:40,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60448 deadline: 1732372120238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:40,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:40,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60476 deadline: 1732372120239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:40,245 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:40,245 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:40,245 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:40,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:40,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:40,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:40,352 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:40,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60492 deadline: 1732372120350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:40,353 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:40,353 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:40,353 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:40,353 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:40,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60468 deadline: 1732372120350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:40,353 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:40,353 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:40,353 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:40,353 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:40,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60448 deadline: 1732372120350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:40,353 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:40,353 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:40,353 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:40,353 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:40,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60476 deadline: 1732372120350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:40,353 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:40,353 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:40,354 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:40,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60470 deadline: 1732372120350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:40,354 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:40,354 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:40,354 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:40,354 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:40,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:40,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60468 deadline: 1732372120560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:40,562 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:40,562 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:40,562 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:40,563 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:40,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60470 deadline: 1732372120560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:40,563 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:40,563 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:40,563 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:40,563 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:40,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60492 deadline: 1732372120560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:40,564 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:40,564 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:40,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:40,564 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:40,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60448 deadline: 1732372120560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:40,564 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:40,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:40,564 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:40,564 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:40,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60476 deadline: 1732372120560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:40,564 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:40,564 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:40,564 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:40,621 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:40,628 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123e6afb821ebe64211abbde642cd16eb77_59fa5c31bb2297940cc7fa9d0ff623a3 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123e6afb821ebe64211abbde642cd16eb77_59fa5c31bb2297940cc7fa9d0ff623a3 2024-11-23T14:27:40,629 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/A/3213e5165612488eb54c9f871f563ef6, store: [table=TestAcidGuarantees family=A region=59fa5c31bb2297940cc7fa9d0ff623a3] 2024-11-23T14:27:40,630 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/A/3213e5165612488eb54c9f871f563ef6 is 175, key is test_row_0/A:col10/1732372060191/Put/seqid=0 2024-11-23T14:27:40,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742420_1596 (size=30955) 2024-11-23T14:27:40,634 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=94, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/A/3213e5165612488eb54c9f871f563ef6 2024-11-23T14:27:40,641 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/B/f007742cef554fd4b1690dbf00ff2c47 is 50, key is test_row_0/B:col10/1732372060191/Put/seqid=0 2024-11-23T14:27:40,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742421_1597 (size=12001) 2024-11-23T14:27:40,652 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/B/f007742cef554fd4b1690dbf00ff2c47 2024-11-23T14:27:40,678 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/C/e1e7ef02a7e24c5d85569097ba02520f is 50, key is test_row_0/C:col10/1732372060191/Put/seqid=0 2024-11-23T14:27:40,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742422_1598 (size=12001) 2024-11-23T14:27:40,692 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/C/e1e7ef02a7e24c5d85569097ba02520f 2024-11-23T14:27:40,698 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/A/3213e5165612488eb54c9f871f563ef6 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/3213e5165612488eb54c9f871f563ef6 2024-11-23T14:27:40,701 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/3213e5165612488eb54c9f871f563ef6, entries=150, sequenceid=94, filesize=30.2 K 2024-11-23T14:27:40,702 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/B/f007742cef554fd4b1690dbf00ff2c47 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/f007742cef554fd4b1690dbf00ff2c47 2024-11-23T14:27:40,706 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/f007742cef554fd4b1690dbf00ff2c47, entries=150, sequenceid=94, filesize=11.7 K 2024-11-23T14:27:40,706 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/C/e1e7ef02a7e24c5d85569097ba02520f as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/e1e7ef02a7e24c5d85569097ba02520f 2024-11-23T14:27:40,709 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/e1e7ef02a7e24c5d85569097ba02520f, entries=150, sequenceid=94, filesize=11.7 K 2024-11-23T14:27:40,710 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 59fa5c31bb2297940cc7fa9d0ff623a3 in 513ms, sequenceid=94, compaction requested=true 2024-11-23T14:27:40,710 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 59fa5c31bb2297940cc7fa9d0ff623a3: 2024-11-23T14:27:40,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 59fa5c31bb2297940cc7fa9d0ff623a3:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T14:27:40,710 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:27:40,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:27:40,710 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:27:40,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 59fa5c31bb2297940cc7fa9d0ff623a3:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T14:27:40,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:27:40,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 59fa5c31bb2297940cc7fa9d0ff623a3:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T14:27:40,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:27:40,712 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92968 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:27:40,712 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1541): 59fa5c31bb2297940cc7fa9d0ff623a3/A is initiating minor compaction (all files) 2024-11-23T14:27:40,712 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 59fa5c31bb2297940cc7fa9d0ff623a3/A in TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:40,712 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/b4c583d41d4b47938ca835dd1bd635d6, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/b28956adc03f49fe85d832ee99a4ff80, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/3213e5165612488eb54c9f871f563ef6] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp, totalSize=90.8 K 2024-11-23T14:27:40,712 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:40,712 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. files: [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/b4c583d41d4b47938ca835dd1bd635d6, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/b28956adc03f49fe85d832ee99a4ff80, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/3213e5165612488eb54c9f871f563ef6] 2024-11-23T14:27:40,712 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:27:40,712 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting b4c583d41d4b47938ca835dd1bd635d6, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1732372057286 2024-11-23T14:27:40,712 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1541): 59fa5c31bb2297940cc7fa9d0ff623a3/B is initiating minor compaction (all files) 2024-11-23T14:27:40,712 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 59fa5c31bb2297940cc7fa9d0ff623a3/B in TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:40,712 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/ebb73acb95214f769491fbfccb36903f, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/dcfa774201a24337986d60ee24efdde8, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/f007742cef554fd4b1690dbf00ff2c47] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp, totalSize=35.3 K 2024-11-23T14:27:40,713 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting b28956adc03f49fe85d832ee99a4ff80, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1732372057973 2024-11-23T14:27:40,713 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting ebb73acb95214f769491fbfccb36903f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1732372057286 2024-11-23T14:27:40,715 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3213e5165612488eb54c9f871f563ef6, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732372060169 2024-11-23T14:27:40,715 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting dcfa774201a24337986d60ee24efdde8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1732372057973 2024-11-23T14:27:40,715 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting f007742cef554fd4b1690dbf00ff2c47, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732372060169 2024-11-23T14:27:40,732 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=59fa5c31bb2297940cc7fa9d0ff623a3] 2024-11-23T14:27:40,741 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 59fa5c31bb2297940cc7fa9d0ff623a3#B#compaction#510 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:27:40,742 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/B/8954bad5b3074923af22fffb8d9867f1 is 50, key is test_row_0/B:col10/1732372060191/Put/seqid=0 2024-11-23T14:27:40,761 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411238a1a81c2349640a5b64ae2846e5a5165_59fa5c31bb2297940cc7fa9d0ff623a3 store=[table=TestAcidGuarantees family=A region=59fa5c31bb2297940cc7fa9d0ff623a3] 2024-11-23T14:27:40,763 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411238a1a81c2349640a5b64ae2846e5a5165_59fa5c31bb2297940cc7fa9d0ff623a3, store=[table=TestAcidGuarantees family=A region=59fa5c31bb2297940cc7fa9d0ff623a3] 2024-11-23T14:27:40,763 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411238a1a81c2349640a5b64ae2846e5a5165_59fa5c31bb2297940cc7fa9d0ff623a3 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=59fa5c31bb2297940cc7fa9d0ff623a3] 2024-11-23T14:27:40,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742423_1599 (size=12207) 2024-11-23T14:27:40,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742424_1600 (size=4469) 2024-11-23T14:27:40,805 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/B/8954bad5b3074923af22fffb8d9867f1 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/8954bad5b3074923af22fffb8d9867f1 2024-11-23T14:27:40,806 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 59fa5c31bb2297940cc7fa9d0ff623a3#A#compaction#509 average throughput is 0.33 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:27:40,807 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/A/1b9b7b6da2aa41f28e6a6141bc5edab8 is 175, key is test_row_0/A:col10/1732372060191/Put/seqid=0 2024-11-23T14:27:40,810 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 59fa5c31bb2297940cc7fa9d0ff623a3/B of 59fa5c31bb2297940cc7fa9d0ff623a3 into 8954bad5b3074923af22fffb8d9867f1(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:27:40,810 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 59fa5c31bb2297940cc7fa9d0ff623a3: 2024-11-23T14:27:40,810 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., storeName=59fa5c31bb2297940cc7fa9d0ff623a3/B, priority=13, startTime=1732372060710; duration=0sec 2024-11-23T14:27:40,810 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:27:40,810 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 59fa5c31bb2297940cc7fa9d0ff623a3:B 2024-11-23T14:27:40,811 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:27:40,812 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:27:40,812 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1541): 59fa5c31bb2297940cc7fa9d0ff623a3/C is initiating minor compaction (all files) 2024-11-23T14:27:40,812 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 59fa5c31bb2297940cc7fa9d0ff623a3/C in TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:40,812 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/4ebe0ca8dc9b494c9730827905c76429, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/8664b2065ba14f07b640dd8aa9bd854a, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/e1e7ef02a7e24c5d85569097ba02520f] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp, totalSize=35.3 K 2024-11-23T14:27:40,825 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ebe0ca8dc9b494c9730827905c76429, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1732372057286 2024-11-23T14:27:40,825 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 8664b2065ba14f07b640dd8aa9bd854a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1732372057973 2024-11-23T14:27:40,825 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting e1e7ef02a7e24c5d85569097ba02520f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732372060169 2024-11-23T14:27:40,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742425_1601 (size=31161) 2024-11-23T14:27:40,840 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/A/1b9b7b6da2aa41f28e6a6141bc5edab8 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/1b9b7b6da2aa41f28e6a6141bc5edab8 2024-11-23T14:27:40,853 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 59fa5c31bb2297940cc7fa9d0ff623a3#C#compaction#511 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:27:40,853 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/C/4dc80c9b40594fe7961b4ceb21c7085e is 50, key is test_row_0/C:col10/1732372060191/Put/seqid=0 2024-11-23T14:27:40,859 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 59fa5c31bb2297940cc7fa9d0ff623a3/A of 59fa5c31bb2297940cc7fa9d0ff623a3 into 1b9b7b6da2aa41f28e6a6141bc5edab8(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:27:40,859 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 59fa5c31bb2297940cc7fa9d0ff623a3: 2024-11-23T14:27:40,859 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., storeName=59fa5c31bb2297940cc7fa9d0ff623a3/A, priority=13, startTime=1732372060710; duration=0sec 2024-11-23T14:27:40,859 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:27:40,859 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 59fa5c31bb2297940cc7fa9d0ff623a3:A 2024-11-23T14:27:40,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742426_1602 (size=12207) 2024-11-23T14:27:40,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-23T14:27:40,871 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T14:27:40,871 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-23T14:27:40,872 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/C/4dc80c9b40594fe7961b4ceb21c7085e as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/4dc80c9b40594fe7961b4ceb21c7085e 2024-11-23T14:27:40,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-23T14:27:40,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-23T14:27:40,874 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T14:27:40,874 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T14:27:40,874 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 59fa5c31bb2297940cc7fa9d0ff623a3 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-23T14:27:40,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on 59fa5c31bb2297940cc7fa9d0ff623a3 2024-11-23T14:27:40,875 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T14:27:40,875 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T14:27:40,882 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 59fa5c31bb2297940cc7fa9d0ff623a3/C of 59fa5c31bb2297940cc7fa9d0ff623a3 into 4dc80c9b40594fe7961b4ceb21c7085e(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:27:40,882 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 59fa5c31bb2297940cc7fa9d0ff623a3: 2024-11-23T14:27:40,882 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., storeName=59fa5c31bb2297940cc7fa9d0ff623a3/C, priority=13, startTime=1732372060711; duration=0sec 2024-11-23T14:27:40,882 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:27:40,882 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 59fa5c31bb2297940cc7fa9d0ff623a3:C 2024-11-23T14:27:40,891 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:40,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60492 deadline: 1732372120885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:40,891 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:40,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60448 deadline: 1732372120885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:40,893 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:40,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:40,893 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:40,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:40,893 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:40,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:40,895 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411239df7de9cee504fb8ba94ca51f3889028_59fa5c31bb2297940cc7fa9d0ff623a3 is 50, key is test_row_0/A:col10/1732372060874/Put/seqid=0 2024-11-23T14:27:40,896 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:40,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60468 deadline: 1732372120892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:40,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:40,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60470 deadline: 1732372120893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:40,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:40,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60476 deadline: 1732372120893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:40,900 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:40,900 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:40,900 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:40,901 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:40,901 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:40,901 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:40,904 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:40,904 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:40,905 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:40,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742427_1603 (size=12154) 2024-11-23T14:27:40,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-23T14:27:41,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:41,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60492 deadline: 1732372121000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:41,002 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:41,002 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:41,002 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:41,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60448 deadline: 1732372121000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:41,002 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:41,002 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:41,002 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:41,002 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:41,011 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:41,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60468 deadline: 1732372121010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:41,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:41,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60476 deadline: 1732372121011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:41,012 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:41,012 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:41,012 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:41,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:41,012 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:41,012 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:41,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60470 deadline: 1732372121011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:41,012 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:41,012 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:41,012 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:41,012 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:41,028 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-23T14:27:41,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:41,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. as already flushing 2024-11-23T14:27:41,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:41,028 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:41,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:41,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:41,180 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-23T14:27:41,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:41,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. as already flushing 2024-11-23T14:27:41,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:41,181 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:41,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:41,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:41,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-23T14:27:41,212 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:41,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60492 deadline: 1732372121210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:41,212 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:41,212 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:41,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60448 deadline: 1732372121210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:41,212 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:41,213 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:41,213 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:41,213 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:41,213 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:41,222 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:41,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60468 deadline: 1732372121220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:41,222 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:41,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60476 deadline: 1732372121220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:41,222 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:41,222 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:41,222 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:41,222 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:41,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60470 deadline: 1732372121220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:41,223 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:41,223 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:41,223 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:41,223 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:41,223 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:41,223 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:41,323 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:41,326 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411239df7de9cee504fb8ba94ca51f3889028_59fa5c31bb2297940cc7fa9d0ff623a3 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411239df7de9cee504fb8ba94ca51f3889028_59fa5c31bb2297940cc7fa9d0ff623a3 2024-11-23T14:27:41,328 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/A/74aa33b45610494dbcb5e7a5c2731685, store: [table=TestAcidGuarantees family=A region=59fa5c31bb2297940cc7fa9d0ff623a3] 2024-11-23T14:27:41,330 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/A/74aa33b45610494dbcb5e7a5c2731685 is 175, key is test_row_0/A:col10/1732372060874/Put/seqid=0 2024-11-23T14:27:41,333 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-23T14:27:41,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:41,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. as already flushing 2024-11-23T14:27:41,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:41,333 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:41,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:41,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:41,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742428_1604 (size=30955) 2024-11-23T14:27:41,337 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=122, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/A/74aa33b45610494dbcb5e7a5c2731685 2024-11-23T14:27:41,343 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/B/2cbc817bbfdf4d1296937bd0c6b32c37 is 50, key is test_row_0/B:col10/1732372060874/Put/seqid=0 2024-11-23T14:27:41,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742429_1605 (size=12001) 2024-11-23T14:27:41,486 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-23T14:27:41,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:41,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. as already flushing 2024-11-23T14:27:41,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:41,486 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:41,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:41,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:41,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-23T14:27:41,521 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:41,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60492 deadline: 1732372121520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:41,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:41,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60448 deadline: 1732372121520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:41,522 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:41,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:41,522 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:41,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:41,522 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:41,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:41,530 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:41,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60468 deadline: 1732372121530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:41,530 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:41,531 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:41,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:41,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60470 deadline: 1732372121530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:41,531 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:41,531 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:41,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60476 deadline: 1732372121530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:41,531 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:41,531 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:41,531 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:41,531 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:41,531 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:41,531 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:41,639 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-23T14:27:41,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:41,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. as already flushing 2024-11-23T14:27:41,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:41,639 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:41,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:41,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:41,750 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=122 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/B/2cbc817bbfdf4d1296937bd0c6b32c37 2024-11-23T14:27:41,759 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/C/92bb65620a8544bfb033933e9ffe7a49 is 50, key is test_row_0/C:col10/1732372060874/Put/seqid=0 2024-11-23T14:27:41,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742430_1606 (size=12001) 2024-11-23T14:27:41,792 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-23T14:27:41,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:41,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. as already flushing 2024-11-23T14:27:41,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:41,793 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:41,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:41,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:41,946 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-23T14:27:41,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:41,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. as already flushing 2024-11-23T14:27:41,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:41,947 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:41,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:41,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:42,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-23T14:27:42,031 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:42,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60492 deadline: 1732372122030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:42,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:42,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60448 deadline: 1732372122030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:42,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:42,032 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:42,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:42,032 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:42,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:42,032 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:42,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:42,041 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:42,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60476 deadline: 1732372122040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:42,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60470 deadline: 1732372122040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:42,042 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:42,042 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:42,042 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:42,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60468 deadline: 1732372122041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:42,042 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:42,042 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:42,042 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:42,042 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:42,042 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:42,042 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:42,042 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:42,053 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-23T14:27:42,053 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-23T14:27:42,100 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-23T14:27:42,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:42,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. as already flushing 2024-11-23T14:27:42,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:42,100 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:42,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:42,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:42,166 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=122 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/C/92bb65620a8544bfb033933e9ffe7a49 2024-11-23T14:27:42,179 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/A/74aa33b45610494dbcb5e7a5c2731685 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/74aa33b45610494dbcb5e7a5c2731685 2024-11-23T14:27:42,183 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/74aa33b45610494dbcb5e7a5c2731685, entries=150, sequenceid=122, filesize=30.2 K 2024-11-23T14:27:42,183 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/B/2cbc817bbfdf4d1296937bd0c6b32c37 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/2cbc817bbfdf4d1296937bd0c6b32c37 2024-11-23T14:27:42,187 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/2cbc817bbfdf4d1296937bd0c6b32c37, entries=150, sequenceid=122, filesize=11.7 K 2024-11-23T14:27:42,188 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/C/92bb65620a8544bfb033933e9ffe7a49 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/92bb65620a8544bfb033933e9ffe7a49 2024-11-23T14:27:42,192 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/92bb65620a8544bfb033933e9ffe7a49, entries=150, sequenceid=122, filesize=11.7 K 2024-11-23T14:27:42,193 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=60.38 KB/61830 for 59fa5c31bb2297940cc7fa9d0ff623a3 in 1319ms, sequenceid=122, compaction requested=false 2024-11-23T14:27:42,193 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 59fa5c31bb2297940cc7fa9d0ff623a3: 2024-11-23T14:27:42,252 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-23T14:27:42,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:42,253 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing 59fa5c31bb2297940cc7fa9d0ff623a3 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-23T14:27:42,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123fcde1044f5c74f4d9b162a91ba40d25c_59fa5c31bb2297940cc7fa9d0ff623a3 is 50, key is test_row_0/A:col10/1732372060889/Put/seqid=0 2024-11-23T14:27:42,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742431_1607 (size=12304) 2024-11-23T14:27:42,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:42,668 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123fcde1044f5c74f4d9b162a91ba40d25c_59fa5c31bb2297940cc7fa9d0ff623a3 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123fcde1044f5c74f4d9b162a91ba40d25c_59fa5c31bb2297940cc7fa9d0ff623a3 2024-11-23T14:27:42,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/A/449f26e910c649ecb57b375b225d4601, store: [table=TestAcidGuarantees family=A region=59fa5c31bb2297940cc7fa9d0ff623a3] 2024-11-23T14:27:42,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/A/449f26e910c649ecb57b375b225d4601 is 175, key is test_row_0/A:col10/1732372060889/Put/seqid=0 2024-11-23T14:27:42,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742432_1608 (size=31105) 2024-11-23T14:27:42,675 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=135, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/A/449f26e910c649ecb57b375b225d4601 2024-11-23T14:27:42,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/B/ab3a75da0f4e498ebc1de46459d34cf3 is 50, key is test_row_0/B:col10/1732372060889/Put/seqid=0 2024-11-23T14:27:42,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742433_1609 (size=12151) 2024-11-23T14:27:43,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-23T14:27:43,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on 59fa5c31bb2297940cc7fa9d0ff623a3 2024-11-23T14:27:43,052 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. as already flushing 2024-11-23T14:27:43,087 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/B/ab3a75da0f4e498ebc1de46459d34cf3 2024-11-23T14:27:43,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/C/bc94a67f48194873b251663a454bbb98 is 50, key is test_row_0/C:col10/1732372060889/Put/seqid=0 2024-11-23T14:27:43,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742434_1610 (size=12151) 2024-11-23T14:27:43,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:43,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60470 deadline: 1732372123111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:43,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:43,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60492 deadline: 1732372123112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:43,117 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:43,117 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:43,117 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:43,117 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:43,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:43,117 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:43,117 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:43,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60476 deadline: 1732372123112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:43,118 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:43,118 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:43,118 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:43,121 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:43,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60468 deadline: 1732372123117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:43,121 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:43,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60448 deadline: 1732372123117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:43,121 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:43,121 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:43,121 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:43,121 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:43,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:43,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:43,221 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:43,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60470 deadline: 1732372123220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:43,221 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:43,221 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:43,221 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:43,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60476 deadline: 1732372123220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:43,227 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:43,227 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:43,227 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:43,227 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:43,227 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:43,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60492 deadline: 1732372123227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:43,228 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:43,228 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:43,228 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:43,232 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:43,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60468 deadline: 1732372123230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:43,232 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:43,232 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:43,233 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:43,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60448 deadline: 1732372123231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:43,233 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:43,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:43,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:43,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:43,432 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:43,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60470 deadline: 1732372123430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:43,432 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:43,432 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:43,432 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:43,433 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:43,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60476 deadline: 1732372123430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:43,434 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:43,434 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:43,434 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:43,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:43,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60492 deadline: 1732372123440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:43,443 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:43,443 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:43,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:43,443 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:43,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60448 deadline: 1732372123440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:43,443 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:43,443 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:43,443 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:43,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:43,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60468 deadline: 1732372123440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:43,443 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:43,443 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:43,443 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:43,495 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/C/bc94a67f48194873b251663a454bbb98 2024-11-23T14:27:43,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/A/449f26e910c649ecb57b375b225d4601 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/449f26e910c649ecb57b375b225d4601 2024-11-23T14:27:43,501 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/449f26e910c649ecb57b375b225d4601, entries=150, sequenceid=135, filesize=30.4 K 2024-11-23T14:27:43,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/B/ab3a75da0f4e498ebc1de46459d34cf3 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/ab3a75da0f4e498ebc1de46459d34cf3 2024-11-23T14:27:43,505 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/ab3a75da0f4e498ebc1de46459d34cf3, entries=150, sequenceid=135, filesize=11.9 K 2024-11-23T14:27:43,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/C/bc94a67f48194873b251663a454bbb98 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/bc94a67f48194873b251663a454bbb98 2024-11-23T14:27:43,509 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/bc94a67f48194873b251663a454bbb98, entries=150, sequenceid=135, filesize=11.9 K 2024-11-23T14:27:43,510 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 59fa5c31bb2297940cc7fa9d0ff623a3 in 1257ms, sequenceid=135, compaction requested=true 2024-11-23T14:27:43,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for 59fa5c31bb2297940cc7fa9d0ff623a3: 2024-11-23T14:27:43,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:43,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-23T14:27:43,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-23T14:27:43,513 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-23T14:27:43,513 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6360 sec 2024-11-23T14:27:43,514 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 2.6420 sec 2024-11-23T14:27:43,745 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T14:27:43,745 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 59fa5c31bb2297940cc7fa9d0ff623a3 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-23T14:27:43,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on 59fa5c31bb2297940cc7fa9d0ff623a3 2024-11-23T14:27:43,761 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:43,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60448 deadline: 1732372123756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:43,762 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:43,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60492 deadline: 1732372123756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:43,762 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:43,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:43,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60468 deadline: 1732372123758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:43,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60476 deadline: 1732372123758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:43,762 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:43,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60470 deadline: 1732372123759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:43,765 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411239249591aaa384ab9881f436bead6f5a5_59fa5c31bb2297940cc7fa9d0ff623a3 is 50, key is test_row_0/A:col10/1732372063115/Put/seqid=0 2024-11-23T14:27:43,768 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:43,768 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:43,768 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:43,768 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:43,768 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:43,768 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:43,768 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:43,768 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:43,768 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:43,769 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:43,769 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:43,769 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:43,769 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:43,769 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:43,769 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:43,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742435_1611 (size=14794) 2024-11-23T14:27:43,870 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:43,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60448 deadline: 1732372123869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:43,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:43,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:43,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:43,882 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:43,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60492 deadline: 1732372123880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:43,883 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:43,883 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:43,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60470 deadline: 1732372123880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:43,883 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:43,883 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:43,883 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:43,883 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:43,883 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:43,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60468 deadline: 1732372123880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:43,883 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:43,883 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:43,883 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:43,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60476 deadline: 1732372123881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:43,883 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:43,883 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:43,884 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:43,884 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:43,884 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:44,083 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:44,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60448 deadline: 1732372124080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:44,083 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:44,083 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:44,083 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:44,094 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:44,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60492 deadline: 1732372124090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:44,094 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:44,094 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:44,094 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:44,095 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:44,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60470 deadline: 1732372124090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:44,095 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:44,095 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:44,095 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:44,095 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:44,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60468 deadline: 1732372124090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:44,095 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:44,095 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:44,095 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:44,095 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:44,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60476 deadline: 1732372124090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:44,096 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:44,096 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:44,096 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:44,181 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:44,183 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411239249591aaa384ab9881f436bead6f5a5_59fa5c31bb2297940cc7fa9d0ff623a3 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411239249591aaa384ab9881f436bead6f5a5_59fa5c31bb2297940cc7fa9d0ff623a3 2024-11-23T14:27:44,185 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/A/4dc72b98072644688f40348257298251, store: [table=TestAcidGuarantees family=A region=59fa5c31bb2297940cc7fa9d0ff623a3] 2024-11-23T14:27:44,185 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/A/4dc72b98072644688f40348257298251 is 175, key is test_row_0/A:col10/1732372063115/Put/seqid=0 2024-11-23T14:27:44,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742436_1612 (size=39749) 2024-11-23T14:27:44,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:44,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60448 deadline: 1732372124389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:44,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:44,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:44,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:44,400 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:44,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60492 deadline: 1732372124400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:44,401 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:44,401 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:44,401 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:44,401 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:44,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60470 deadline: 1732372124400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:44,401 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:44,401 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:44,401 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:44,401 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:44,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60468 deadline: 1732372124400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:44,401 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:44,401 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:44,401 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:44,401 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:44,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60476 deadline: 1732372124400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:44,402 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:44,402 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:44,402 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:44,588 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=162, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/A/4dc72b98072644688f40348257298251 2024-11-23T14:27:44,594 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/B/bb5eefb597174834a05e4b085ac0d7bc is 50, key is test_row_0/B:col10/1732372063115/Put/seqid=0 2024-11-23T14:27:44,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742437_1613 (size=12151) 2024-11-23T14:27:44,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:44,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60448 deadline: 1732372124900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:44,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:44,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:44,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:44,913 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:44,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60492 deadline: 1732372124910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:44,913 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:44,913 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:44,913 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:44,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60470 deadline: 1732372124910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:44,914 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:44,914 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:44,914 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:44,914 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:44,914 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:44,914 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:44,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60468 deadline: 1732372124910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:44,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60476 deadline: 1732372124910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:44,914 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:44,914 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:44,914 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:44,914 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:44,914 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:44,914 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:45,010 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=162 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/B/bb5eefb597174834a05e4b085ac0d7bc 2024-11-23T14:27:45,016 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/C/741430a73c964c31a1511d257d5624d5 is 50, key is test_row_0/C:col10/1732372063115/Put/seqid=0 2024-11-23T14:27:45,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742438_1614 (size=12151) 2024-11-23T14:27:45,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-23T14:27:45,040 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T14:27:45,041 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-23T14:27:45,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-11-23T14:27:45,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-23T14:27:45,043 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T14:27:45,043 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T14:27:45,043 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T14:27:45,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-23T14:27:45,196 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-23T14:27:45,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:45,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. as already flushing 2024-11-23T14:27:45,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:45,196 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:45,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:45,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:45,350 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-23T14:27:45,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:45,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. as already flushing 2024-11-23T14:27:45,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:45,351 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:45,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:45,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:45,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-23T14:27:45,426 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=162 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/C/741430a73c964c31a1511d257d5624d5 2024-11-23T14:27:45,435 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/A/4dc72b98072644688f40348257298251 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/4dc72b98072644688f40348257298251 2024-11-23T14:27:45,437 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/4dc72b98072644688f40348257298251, entries=200, sequenceid=162, filesize=38.8 K 2024-11-23T14:27:45,438 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/B/bb5eefb597174834a05e4b085ac0d7bc as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/bb5eefb597174834a05e4b085ac0d7bc 2024-11-23T14:27:45,440 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/bb5eefb597174834a05e4b085ac0d7bc, entries=150, sequenceid=162, filesize=11.9 K 2024-11-23T14:27:45,441 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/C/741430a73c964c31a1511d257d5624d5 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/741430a73c964c31a1511d257d5624d5 2024-11-23T14:27:45,443 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/741430a73c964c31a1511d257d5624d5, entries=150, sequenceid=162, filesize=11.9 K 2024-11-23T14:27:45,444 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=46.96 KB/48090 for 59fa5c31bb2297940cc7fa9d0ff623a3 in 1699ms, sequenceid=162, compaction requested=true 2024-11-23T14:27:45,444 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 59fa5c31bb2297940cc7fa9d0ff623a3: 2024-11-23T14:27:45,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 59fa5c31bb2297940cc7fa9d0ff623a3:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T14:27:45,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:27:45,444 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T14:27:45,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 59fa5c31bb2297940cc7fa9d0ff623a3:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T14:27:45,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:27:45,444 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T14:27:45,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 59fa5c31bb2297940cc7fa9d0ff623a3:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T14:27:45,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:27:45,446 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 132970 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T14:27:45,446 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48510 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T14:27:45,446 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1541): 59fa5c31bb2297940cc7fa9d0ff623a3/A is initiating minor compaction (all files) 2024-11-23T14:27:45,446 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1541): 59fa5c31bb2297940cc7fa9d0ff623a3/B is initiating minor compaction (all files) 2024-11-23T14:27:45,446 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 59fa5c31bb2297940cc7fa9d0ff623a3/A in TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:45,446 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 59fa5c31bb2297940cc7fa9d0ff623a3/B in TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:45,446 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/1b9b7b6da2aa41f28e6a6141bc5edab8, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/74aa33b45610494dbcb5e7a5c2731685, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/449f26e910c649ecb57b375b225d4601, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/4dc72b98072644688f40348257298251] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp, totalSize=129.9 K 2024-11-23T14:27:45,446 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/8954bad5b3074923af22fffb8d9867f1, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/2cbc817bbfdf4d1296937bd0c6b32c37, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/ab3a75da0f4e498ebc1de46459d34cf3, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/bb5eefb597174834a05e4b085ac0d7bc] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp, totalSize=47.4 K 2024-11-23T14:27:45,446 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:45,446 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. files: [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/1b9b7b6da2aa41f28e6a6141bc5edab8, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/74aa33b45610494dbcb5e7a5c2731685, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/449f26e910c649ecb57b375b225d4601, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/4dc72b98072644688f40348257298251] 2024-11-23T14:27:45,447 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b9b7b6da2aa41f28e6a6141bc5edab8, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732372060169 2024-11-23T14:27:45,447 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 8954bad5b3074923af22fffb8d9867f1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732372060169 2024-11-23T14:27:45,447 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 74aa33b45610494dbcb5e7a5c2731685, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1732372060236 2024-11-23T14:27:45,447 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 2cbc817bbfdf4d1296937bd0c6b32c37, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1732372060236 2024-11-23T14:27:45,447 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 449f26e910c649ecb57b375b225d4601, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1732372060883 2024-11-23T14:27:45,447 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting ab3a75da0f4e498ebc1de46459d34cf3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1732372060883 2024-11-23T14:27:45,447 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4dc72b98072644688f40348257298251, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1732372063110 2024-11-23T14:27:45,447 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting bb5eefb597174834a05e4b085ac0d7bc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1732372063110 2024-11-23T14:27:45,455 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 59fa5c31bb2297940cc7fa9d0ff623a3#B#compaction#521 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:27:45,456 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/B/2b3ae18e14a34736bb118604de5793c3 is 50, key is test_row_0/B:col10/1732372063115/Put/seqid=0 2024-11-23T14:27:45,459 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=59fa5c31bb2297940cc7fa9d0ff623a3] 2024-11-23T14:27:45,460 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241123c953a54e1caa4fedaf16e9913ab3ceb9_59fa5c31bb2297940cc7fa9d0ff623a3 store=[table=TestAcidGuarantees family=A region=59fa5c31bb2297940cc7fa9d0ff623a3] 2024-11-23T14:27:45,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742439_1615 (size=12493) 2024-11-23T14:27:45,463 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241123c953a54e1caa4fedaf16e9913ab3ceb9_59fa5c31bb2297940cc7fa9d0ff623a3, store=[table=TestAcidGuarantees family=A region=59fa5c31bb2297940cc7fa9d0ff623a3] 2024-11-23T14:27:45,463 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123c953a54e1caa4fedaf16e9913ab3ceb9_59fa5c31bb2297940cc7fa9d0ff623a3 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=59fa5c31bb2297940cc7fa9d0ff623a3] 2024-11-23T14:27:45,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742440_1616 (size=4469) 2024-11-23T14:27:45,468 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/B/2b3ae18e14a34736bb118604de5793c3 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/2b3ae18e14a34736bb118604de5793c3 2024-11-23T14:27:45,472 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 59fa5c31bb2297940cc7fa9d0ff623a3/B of 59fa5c31bb2297940cc7fa9d0ff623a3 into 2b3ae18e14a34736bb118604de5793c3(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:27:45,473 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 59fa5c31bb2297940cc7fa9d0ff623a3: 2024-11-23T14:27:45,473 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., storeName=59fa5c31bb2297940cc7fa9d0ff623a3/B, priority=12, startTime=1732372065444; duration=0sec 2024-11-23T14:27:45,473 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:27:45,473 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 59fa5c31bb2297940cc7fa9d0ff623a3:B 2024-11-23T14:27:45,473 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T14:27:45,474 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48510 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T14:27:45,474 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1541): 59fa5c31bb2297940cc7fa9d0ff623a3/C is initiating minor compaction (all files) 2024-11-23T14:27:45,474 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 59fa5c31bb2297940cc7fa9d0ff623a3/C in TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:45,474 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/4dc80c9b40594fe7961b4ceb21c7085e, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/92bb65620a8544bfb033933e9ffe7a49, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/bc94a67f48194873b251663a454bbb98, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/741430a73c964c31a1511d257d5624d5] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp, totalSize=47.4 K 2024-11-23T14:27:45,474 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 4dc80c9b40594fe7961b4ceb21c7085e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732372060169 2024-11-23T14:27:45,474 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 92bb65620a8544bfb033933e9ffe7a49, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1732372060236 2024-11-23T14:27:45,474 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting bc94a67f48194873b251663a454bbb98, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1732372060883 2024-11-23T14:27:45,475 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 741430a73c964c31a1511d257d5624d5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1732372063110 2024-11-23T14:27:45,484 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 59fa5c31bb2297940cc7fa9d0ff623a3#C#compaction#523 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:27:45,484 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/C/ddd5a451897048c19e89a9cf95b2490b is 50, key is test_row_0/C:col10/1732372063115/Put/seqid=0 2024-11-23T14:27:45,504 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-23T14:27:45,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:45,505 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing 59fa5c31bb2297940cc7fa9d0ff623a3 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-23T14:27:45,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742441_1617 (size=12493) 2024-11-23T14:27:45,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123860151f05ea849dda17876530533af8e_59fa5c31bb2297940cc7fa9d0ff623a3 is 50, key is test_row_0/A:col10/1732372063756/Put/seqid=0 2024-11-23T14:27:45,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742442_1618 (size=12304) 2024-11-23T14:27:45,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-23T14:27:45,869 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 59fa5c31bb2297940cc7fa9d0ff623a3#A#compaction#522 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:27:45,869 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/A/1e899a3f480b4a9b83d0d6e4dfa3a3a4 is 175, key is test_row_0/A:col10/1732372063115/Put/seqid=0 2024-11-23T14:27:45,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742443_1619 (size=31447) 2024-11-23T14:27:45,900 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/C/ddd5a451897048c19e89a9cf95b2490b as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/ddd5a451897048c19e89a9cf95b2490b 2024-11-23T14:27:45,904 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 59fa5c31bb2297940cc7fa9d0ff623a3/C of 59fa5c31bb2297940cc7fa9d0ff623a3 into ddd5a451897048c19e89a9cf95b2490b(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:27:45,904 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 59fa5c31bb2297940cc7fa9d0ff623a3: 2024-11-23T14:27:45,904 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., storeName=59fa5c31bb2297940cc7fa9d0ff623a3/C, priority=12, startTime=1732372065444; duration=0sec 2024-11-23T14:27:45,904 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:27:45,904 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 59fa5c31bb2297940cc7fa9d0ff623a3:C 2024-11-23T14:27:45,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on 59fa5c31bb2297940cc7fa9d0ff623a3 2024-11-23T14:27:45,911 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. as already flushing 2024-11-23T14:27:45,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:45,939 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123860151f05ea849dda17876530533af8e_59fa5c31bb2297940cc7fa9d0ff623a3 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123860151f05ea849dda17876530533af8e_59fa5c31bb2297940cc7fa9d0ff623a3 2024-11-23T14:27:45,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/A/2120a4a3dfc74f5598eb0f6d76abb3b5, store: [table=TestAcidGuarantees family=A region=59fa5c31bb2297940cc7fa9d0ff623a3] 2024-11-23T14:27:45,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/A/2120a4a3dfc74f5598eb0f6d76abb3b5 is 175, key is test_row_0/A:col10/1732372063756/Put/seqid=0 2024-11-23T14:27:45,947 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:45,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60476 deadline: 1732372125942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:45,947 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:45,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60448 deadline: 1732372125946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:45,948 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:45,948 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:45,948 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:45,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:45,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60492 deadline: 1732372125948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:45,948 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:45,948 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:45,948 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:45,948 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:45,948 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:45,948 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:45,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:45,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60468 deadline: 1732372125948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:45,948 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:45,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:45,949 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:45,949 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:45,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60470 deadline: 1732372125948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:45,949 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:45,949 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:45,949 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:45,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742444_1620 (size=31105) 2024-11-23T14:27:45,954 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=173, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/A/2120a4a3dfc74f5598eb0f6d76abb3b5 2024-11-23T14:27:45,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/B/77e7f8fd88a046828f214d716fcb3fb3 is 50, key is test_row_0/B:col10/1732372063756/Put/seqid=0 2024-11-23T14:27:45,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742445_1621 (size=12151) 2024-11-23T14:27:46,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:46,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60476 deadline: 1732372126053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:46,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:46,054 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:46,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60448 deadline: 1732372126053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:46,054 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:46,055 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:46,055 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:46,055 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:46,055 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:46,060 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:46,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60468 deadline: 1732372126060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:46,060 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:46,060 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:46,060 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:46,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60492 deadline: 1732372126060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:46,060 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:46,061 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:46,061 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:46,061 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:46,061 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:46,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60470 deadline: 1732372126060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:46,061 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:46,061 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:46,061 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:46,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-23T14:27:46,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:46,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60476 deadline: 1732372126260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:46,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:46,262 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:46,262 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:46,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60448 deadline: 1732372126260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:46,262 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:46,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:46,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:46,263 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:46,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:46,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60468 deadline: 1732372126270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:46,274 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:46,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:46,274 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:46,274 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:46,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60470 deadline: 1732372126270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:46,274 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:46,274 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:46,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:46,274 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:46,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60492 deadline: 1732372126270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:46,274 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:46,274 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:46,274 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:46,280 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/A/1e899a3f480b4a9b83d0d6e4dfa3a3a4 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/1e899a3f480b4a9b83d0d6e4dfa3a3a4 2024-11-23T14:27:46,288 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 59fa5c31bb2297940cc7fa9d0ff623a3/A of 59fa5c31bb2297940cc7fa9d0ff623a3 into 1e899a3f480b4a9b83d0d6e4dfa3a3a4(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:27:46,288 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 59fa5c31bb2297940cc7fa9d0ff623a3: 2024-11-23T14:27:46,288 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., storeName=59fa5c31bb2297940cc7fa9d0ff623a3/A, priority=12, startTime=1732372065444; duration=0sec 2024-11-23T14:27:46,288 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:27:46,288 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 59fa5c31bb2297940cc7fa9d0ff623a3:A 2024-11-23T14:27:46,364 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/B/77e7f8fd88a046828f214d716fcb3fb3 2024-11-23T14:27:46,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/C/a3e554e5af944f579222ce5500d63cc0 is 50, key is test_row_0/C:col10/1732372063756/Put/seqid=0 2024-11-23T14:27:46,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742446_1622 (size=12151) 2024-11-23T14:27:46,571 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:46,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60476 deadline: 1732372126570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:46,572 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:46,572 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:46,572 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:46,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60448 deadline: 1732372126570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:46,572 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:46,572 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:46,572 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:46,572 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:46,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:46,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60468 deadline: 1732372126580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:46,583 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:46,583 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:46,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:46,583 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:46,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60470 deadline: 1732372126580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:46,584 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:46,584 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:46,584 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:46,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:46,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60492 deadline: 1732372126580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:46,584 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:46,584 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:46,584 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:46,776 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/C/a3e554e5af944f579222ce5500d63cc0 2024-11-23T14:27:46,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/A/2120a4a3dfc74f5598eb0f6d76abb3b5 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/2120a4a3dfc74f5598eb0f6d76abb3b5 2024-11-23T14:27:46,783 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/2120a4a3dfc74f5598eb0f6d76abb3b5, entries=150, sequenceid=173, filesize=30.4 K 2024-11-23T14:27:46,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/B/77e7f8fd88a046828f214d716fcb3fb3 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/77e7f8fd88a046828f214d716fcb3fb3 2024-11-23T14:27:46,787 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/77e7f8fd88a046828f214d716fcb3fb3, entries=150, sequenceid=173, filesize=11.9 K 2024-11-23T14:27:46,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/C/a3e554e5af944f579222ce5500d63cc0 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/a3e554e5af944f579222ce5500d63cc0 2024-11-23T14:27:46,793 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/a3e554e5af944f579222ce5500d63cc0, entries=150, sequenceid=173, filesize=11.9 K 2024-11-23T14:27:46,794 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=161.02 KB/164880 for 59fa5c31bb2297940cc7fa9d0ff623a3 in 1290ms, sequenceid=173, compaction requested=false 2024-11-23T14:27:46,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for 59fa5c31bb2297940cc7fa9d0ff623a3: 2024-11-23T14:27:46,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:46,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-11-23T14:27:46,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-11-23T14:27:46,797 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-23T14:27:46,797 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7520 sec 2024-11-23T14:27:46,799 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 1.7570 sec 2024-11-23T14:27:47,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on 59fa5c31bb2297940cc7fa9d0ff623a3 2024-11-23T14:27:47,082 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T14:27:47,082 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 59fa5c31bb2297940cc7fa9d0ff623a3 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-23T14:27:47,091 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:47,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60476 deadline: 1732372127089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:47,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:47,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60448 deadline: 1732372127090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:47,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:47,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60468 deadline: 1732372127090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:47,092 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:47,092 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:47,092 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:47,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:47,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60470 deadline: 1732372127091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:47,093 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:47,093 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:47,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60492 deadline: 1732372127091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:47,093 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:47,093 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:47,093 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:47,093 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:47,093 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:47,094 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123ebedf3bbbd654d9b9a91ed6b5470cc49_59fa5c31bb2297940cc7fa9d0ff623a3 is 50, key is test_row_0/A:col10/1732372067081/Put/seqid=0 2024-11-23T14:27:47,100 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:47,100 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:47,100 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:47,100 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:47,100 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:47,100 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:47,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742447_1623 (size=12304) 2024-11-23T14:27:47,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-23T14:27:47,190 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T14:27:47,190 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-23T14:27:47,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees 2024-11-23T14:27:47,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-23T14:27:47,192 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T14:27:47,193 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T14:27:47,193 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T14:27:47,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:47,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60448 deadline: 1732372127200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:47,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:47,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:47,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:47,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:47,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60476 deadline: 1732372127210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:47,212 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:47,212 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:47,212 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:47,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-23T14:27:47,345 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-23T14:27:47,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:47,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. as already flushing 2024-11-23T14:27:47,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:47,345 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:47,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:47,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:47,411 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:47,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60448 deadline: 1732372127410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:47,412 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:47,412 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:47,412 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:47,420 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:47,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60476 deadline: 1732372127420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:47,421 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:47,421 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:47,421 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:47,497 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-23T14:27:47,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:47,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. as already flushing 2024-11-23T14:27:47,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:47,498 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:47,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:47,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:47,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-23T14:27:47,515 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:47,518 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123ebedf3bbbd654d9b9a91ed6b5470cc49_59fa5c31bb2297940cc7fa9d0ff623a3 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123ebedf3bbbd654d9b9a91ed6b5470cc49_59fa5c31bb2297940cc7fa9d0ff623a3 2024-11-23T14:27:47,518 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/A/ce5a8064bdea4ee8b3158a27940a8c46, store: [table=TestAcidGuarantees family=A region=59fa5c31bb2297940cc7fa9d0ff623a3] 2024-11-23T14:27:47,519 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/A/ce5a8064bdea4ee8b3158a27940a8c46 is 175, key is test_row_0/A:col10/1732372067081/Put/seqid=0 2024-11-23T14:27:47,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742448_1624 (size=31105) 2024-11-23T14:27:47,650 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-23T14:27:47,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:47,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. as already flushing 2024-11-23T14:27:47,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:47,650 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:47,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:47,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:47,722 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:47,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60448 deadline: 1732372127720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:47,723 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:47,723 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:47,723 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:47,732 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:47,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60476 deadline: 1732372127729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:47,733 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:47,733 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:47,733 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:47,803 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-23T14:27:47,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:47,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. as already flushing 2024-11-23T14:27:47,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:47,803 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:47,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:47,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:47,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-23T14:27:47,926 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=203, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/A/ce5a8064bdea4ee8b3158a27940a8c46 2024-11-23T14:27:47,932 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/B/b55417cdb155411c9e8942d41c50565d is 50, key is test_row_0/B:col10/1732372067081/Put/seqid=0 2024-11-23T14:27:47,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742449_1625 (size=12151) 2024-11-23T14:27:47,955 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-23T14:27:47,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:47,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. as already flushing 2024-11-23T14:27:47,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:47,956 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:47,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:47,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:48,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:48,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60470 deadline: 1732372128100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:48,102 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to c6f7b1897123,35385,1732371902767 for 'test_row_0' in 59fa5c31bb2297940cc7fa9d0ff623a3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2154 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-23T14:27:48,103 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:48,103 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:48,103 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:48,108 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-23T14:27:48,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:48,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. as already flushing 2024-11-23T14:27:48,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:48,108 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:48,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:48,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:48,112 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:48,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60492 deadline: 1732372128110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:48,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:48,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60468 deadline: 1732372128110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:48,113 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to c6f7b1897123,35385,1732371902767 for 'test_row_1' in 59fa5c31bb2297940cc7fa9d0ff623a3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-23T14:27:48,113 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:48,114 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:48,114 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:48,113 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to c6f7b1897123,35385,1732371902767 for 'test_row_0' in 59fa5c31bb2297940cc7fa9d0ff623a3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-23T14:27:48,114 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:48,114 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:48,114 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:48,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:48,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60448 deadline: 1732372128230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:48,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:48,232 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:48,232 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:48,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:48,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60476 deadline: 1732372128240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:48,244 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:48,244 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:48,244 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:48,261 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-23T14:27:48,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:48,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. as already flushing 2024-11-23T14:27:48,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:48,261 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:48,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:48,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:48,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-23T14:27:48,336 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/B/b55417cdb155411c9e8942d41c50565d 2024-11-23T14:27:48,343 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/C/7c6a4e542bbf4e7d84e13fec89c6e1ec is 50, key is test_row_0/C:col10/1732372067081/Put/seqid=0 2024-11-23T14:27:48,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742450_1626 (size=12151) 2024-11-23T14:27:48,413 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-23T14:27:48,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:48,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. as already flushing 2024-11-23T14:27:48,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:48,414 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:48,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:48,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:48,566 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-23T14:27:48,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:48,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. as already flushing 2024-11-23T14:27:48,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:48,567 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:48,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:48,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:48,719 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-23T14:27:48,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:48,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. as already flushing 2024-11-23T14:27:48,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:48,719 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:48,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:48,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:48,748 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/C/7c6a4e542bbf4e7d84e13fec89c6e1ec 2024-11-23T14:27:48,752 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/A/ce5a8064bdea4ee8b3158a27940a8c46 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/ce5a8064bdea4ee8b3158a27940a8c46 2024-11-23T14:27:48,755 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/ce5a8064bdea4ee8b3158a27940a8c46, entries=150, sequenceid=203, filesize=30.4 K 2024-11-23T14:27:48,756 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/B/b55417cdb155411c9e8942d41c50565d as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/b55417cdb155411c9e8942d41c50565d 2024-11-23T14:27:48,761 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/b55417cdb155411c9e8942d41c50565d, entries=150, sequenceid=203, filesize=11.9 K 2024-11-23T14:27:48,762 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/C/7c6a4e542bbf4e7d84e13fec89c6e1ec as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/7c6a4e542bbf4e7d84e13fec89c6e1ec 2024-11-23T14:27:48,765 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/7c6a4e542bbf4e7d84e13fec89c6e1ec, entries=150, sequenceid=203, filesize=11.9 K 2024-11-23T14:27:48,766 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for 59fa5c31bb2297940cc7fa9d0ff623a3 in 1684ms, sequenceid=203, compaction requested=true 2024-11-23T14:27:48,766 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 59fa5c31bb2297940cc7fa9d0ff623a3: 2024-11-23T14:27:48,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 59fa5c31bb2297940cc7fa9d0ff623a3:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T14:27:48,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:27:48,766 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:27:48,766 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:27:48,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 59fa5c31bb2297940cc7fa9d0ff623a3:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T14:27:48,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:27:48,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 59fa5c31bb2297940cc7fa9d0ff623a3:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T14:27:48,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:27:48,768 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:27:48,768 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1541): 59fa5c31bb2297940cc7fa9d0ff623a3/B is initiating minor compaction (all files) 2024-11-23T14:27:48,768 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 59fa5c31bb2297940cc7fa9d0ff623a3/B in TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:48,768 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/2b3ae18e14a34736bb118604de5793c3, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/77e7f8fd88a046828f214d716fcb3fb3, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/b55417cdb155411c9e8942d41c50565d] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp, totalSize=35.9 K 2024-11-23T14:27:48,768 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93657 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:27:48,768 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1541): 59fa5c31bb2297940cc7fa9d0ff623a3/A is initiating minor compaction (all files) 2024-11-23T14:27:48,769 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 59fa5c31bb2297940cc7fa9d0ff623a3/A in TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:48,769 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/1e899a3f480b4a9b83d0d6e4dfa3a3a4, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/2120a4a3dfc74f5598eb0f6d76abb3b5, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/ce5a8064bdea4ee8b3158a27940a8c46] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp, totalSize=91.5 K 2024-11-23T14:27:48,769 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:48,769 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. files: [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/1e899a3f480b4a9b83d0d6e4dfa3a3a4, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/2120a4a3dfc74f5598eb0f6d76abb3b5, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/ce5a8064bdea4ee8b3158a27940a8c46] 2024-11-23T14:27:48,769 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b3ae18e14a34736bb118604de5793c3, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1732372063110 2024-11-23T14:27:48,769 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e899a3f480b4a9b83d0d6e4dfa3a3a4, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1732372063110 2024-11-23T14:27:48,769 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2120a4a3dfc74f5598eb0f6d76abb3b5, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1732372063748 2024-11-23T14:27:48,769 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 77e7f8fd88a046828f214d716fcb3fb3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1732372063748 2024-11-23T14:27:48,770 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce5a8064bdea4ee8b3158a27940a8c46, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1732372065939 2024-11-23T14:27:48,770 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting b55417cdb155411c9e8942d41c50565d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1732372065939 2024-11-23T14:27:48,776 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 59fa5c31bb2297940cc7fa9d0ff623a3#B#compaction#530 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:27:48,777 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/B/0c6163bf21d04f04baf227da10ebadd3 is 50, key is test_row_0/B:col10/1732372067081/Put/seqid=0 2024-11-23T14:27:48,785 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=59fa5c31bb2297940cc7fa9d0ff623a3] 2024-11-23T14:27:48,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742451_1627 (size=12595) 2024-11-23T14:27:48,793 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241123d5642065c803486b80d4e29c47bb14eb_59fa5c31bb2297940cc7fa9d0ff623a3 store=[table=TestAcidGuarantees family=A region=59fa5c31bb2297940cc7fa9d0ff623a3] 2024-11-23T14:27:48,794 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241123d5642065c803486b80d4e29c47bb14eb_59fa5c31bb2297940cc7fa9d0ff623a3, store=[table=TestAcidGuarantees family=A region=59fa5c31bb2297940cc7fa9d0ff623a3] 2024-11-23T14:27:48,794 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123d5642065c803486b80d4e29c47bb14eb_59fa5c31bb2297940cc7fa9d0ff623a3 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=59fa5c31bb2297940cc7fa9d0ff623a3] 2024-11-23T14:27:48,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742452_1628 (size=4469) 2024-11-23T14:27:48,872 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-23T14:27:48,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:48,872 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2902): Flushing 59fa5c31bb2297940cc7fa9d0ff623a3 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-23T14:27:48,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411231573ad2935064781b794fbd2d3673792_59fa5c31bb2297940cc7fa9d0ff623a3 is 50, key is test_row_0/A:col10/1732372067089/Put/seqid=0 2024-11-23T14:27:48,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742453_1629 (size=12304) 2024-11-23T14:27:48,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:48,905 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411231573ad2935064781b794fbd2d3673792_59fa5c31bb2297940cc7fa9d0ff623a3 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411231573ad2935064781b794fbd2d3673792_59fa5c31bb2297940cc7fa9d0ff623a3 2024-11-23T14:27:48,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/A/0f325658aba0481286dd38abe30194cc, store: [table=TestAcidGuarantees family=A region=59fa5c31bb2297940cc7fa9d0ff623a3] 2024-11-23T14:27:48,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/A/0f325658aba0481286dd38abe30194cc is 175, key is test_row_0/A:col10/1732372067089/Put/seqid=0 2024-11-23T14:27:48,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742454_1630 (size=31105) 2024-11-23T14:27:49,201 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/B/0c6163bf21d04f04baf227da10ebadd3 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/0c6163bf21d04f04baf227da10ebadd3 2024-11-23T14:27:49,210 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 59fa5c31bb2297940cc7fa9d0ff623a3/B of 59fa5c31bb2297940cc7fa9d0ff623a3 into 0c6163bf21d04f04baf227da10ebadd3(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:27:49,210 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 59fa5c31bb2297940cc7fa9d0ff623a3: 2024-11-23T14:27:49,210 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., storeName=59fa5c31bb2297940cc7fa9d0ff623a3/B, priority=13, startTime=1732372068766; duration=0sec 2024-11-23T14:27:49,210 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:27:49,210 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 59fa5c31bb2297940cc7fa9d0ff623a3:B 2024-11-23T14:27:49,211 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:27:49,211 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:27:49,211 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1541): 59fa5c31bb2297940cc7fa9d0ff623a3/C is initiating minor compaction (all files) 2024-11-23T14:27:49,211 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 59fa5c31bb2297940cc7fa9d0ff623a3/C in TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:49,211 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/ddd5a451897048c19e89a9cf95b2490b, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/a3e554e5af944f579222ce5500d63cc0, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/7c6a4e542bbf4e7d84e13fec89c6e1ec] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp, totalSize=35.9 K 2024-11-23T14:27:49,212 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting ddd5a451897048c19e89a9cf95b2490b, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1732372063110 2024-11-23T14:27:49,212 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting a3e554e5af944f579222ce5500d63cc0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1732372063748 2024-11-23T14:27:49,212 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c6a4e542bbf4e7d84e13fec89c6e1ec, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1732372065939 2024-11-23T14:27:49,231 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 59fa5c31bb2297940cc7fa9d0ff623a3#C#compaction#533 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:27:49,231 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/C/c4d727d1f5124394829ddaccf4b8ee67 is 50, key is test_row_0/C:col10/1732372067081/Put/seqid=0 2024-11-23T14:27:49,234 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 59fa5c31bb2297940cc7fa9d0ff623a3#A#compaction#531 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:27:49,234 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/A/66ff6aeeeb674cb29adac00f298dd2c9 is 175, key is test_row_0/A:col10/1732372067081/Put/seqid=0 2024-11-23T14:27:49,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on 59fa5c31bb2297940cc7fa9d0ff623a3 2024-11-23T14:27:49,247 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. as already flushing 2024-11-23T14:27:49,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742455_1631 (size=12595) 2024-11-23T14:27:49,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742456_1632 (size=31549) 2024-11-23T14:27:49,260 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/C/c4d727d1f5124394829ddaccf4b8ee67 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/c4d727d1f5124394829ddaccf4b8ee67 2024-11-23T14:27:49,273 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 59fa5c31bb2297940cc7fa9d0ff623a3/C of 59fa5c31bb2297940cc7fa9d0ff623a3 into c4d727d1f5124394829ddaccf4b8ee67(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:27:49,273 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 59fa5c31bb2297940cc7fa9d0ff623a3: 2024-11-23T14:27:49,273 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., storeName=59fa5c31bb2297940cc7fa9d0ff623a3/C, priority=13, startTime=1732372068766; duration=0sec 2024-11-23T14:27:49,273 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:27:49,273 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 59fa5c31bb2297940cc7fa9d0ff623a3:C 2024-11-23T14:27:49,316 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:49,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60476 deadline: 1732372129313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:49,316 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:49,316 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:49,316 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:49,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:49,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60448 deadline: 1732372129316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:49,319 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:49,319 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:49,319 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:49,322 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=211, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/A/0f325658aba0481286dd38abe30194cc 2024-11-23T14:27:49,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/B/49f255242a6244e6a3ae655bfe087ed1 is 50, key is test_row_0/B:col10/1732372067089/Put/seqid=0 2024-11-23T14:27:49,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742457_1633 (size=12151) 2024-11-23T14:27:49,335 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/B/49f255242a6244e6a3ae655bfe087ed1 2024-11-23T14:27:49,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-23T14:27:49,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/C/9f3e78c2bde44efc95dd7962ab74b55b is 50, key is test_row_0/C:col10/1732372067089/Put/seqid=0 2024-11-23T14:27:49,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742458_1634 (size=12151) 2024-11-23T14:27:49,422 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:49,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60476 deadline: 1732372129420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:49,423 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:49,423 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:49,423 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:49,432 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:49,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60448 deadline: 1732372129430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:49,432 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:49,432 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:49,432 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:49,633 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:49,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60476 deadline: 1732372129630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:49,634 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:49,634 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:49,634 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:49,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:49,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60448 deadline: 1732372129640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:49,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:49,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:49,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:49,661 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/A/66ff6aeeeb674cb29adac00f298dd2c9 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/66ff6aeeeb674cb29adac00f298dd2c9 2024-11-23T14:27:49,665 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 59fa5c31bb2297940cc7fa9d0ff623a3/A of 59fa5c31bb2297940cc7fa9d0ff623a3 into 66ff6aeeeb674cb29adac00f298dd2c9(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:27:49,665 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 59fa5c31bb2297940cc7fa9d0ff623a3: 2024-11-23T14:27:49,665 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., storeName=59fa5c31bb2297940cc7fa9d0ff623a3/A, priority=13, startTime=1732372068766; duration=0sec 2024-11-23T14:27:49,665 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:27:49,666 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 59fa5c31bb2297940cc7fa9d0ff623a3:A 2024-11-23T14:27:49,746 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/C/9f3e78c2bde44efc95dd7962ab74b55b 2024-11-23T14:27:49,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/A/0f325658aba0481286dd38abe30194cc as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/0f325658aba0481286dd38abe30194cc 2024-11-23T14:27:49,753 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/0f325658aba0481286dd38abe30194cc, entries=150, sequenceid=211, filesize=30.4 K 2024-11-23T14:27:49,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/B/49f255242a6244e6a3ae655bfe087ed1 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/49f255242a6244e6a3ae655bfe087ed1 2024-11-23T14:27:49,757 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/49f255242a6244e6a3ae655bfe087ed1, entries=150, sequenceid=211, filesize=11.9 K 2024-11-23T14:27:49,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/C/9f3e78c2bde44efc95dd7962ab74b55b as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/9f3e78c2bde44efc95dd7962ab74b55b 2024-11-23T14:27:49,761 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/9f3e78c2bde44efc95dd7962ab74b55b, entries=150, sequenceid=211, filesize=11.9 K 2024-11-23T14:27:49,761 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=167.72 KB/171750 for 59fa5c31bb2297940cc7fa9d0ff623a3 in 889ms, sequenceid=211, compaction requested=false 2024-11-23T14:27:49,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2603): Flush status journal for 59fa5c31bb2297940cc7fa9d0ff623a3: 2024-11-23T14:27:49,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:49,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=165 2024-11-23T14:27:49,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster(4169): Remote procedure done, pid=165 2024-11-23T14:27:49,764 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-23T14:27:49,764 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5700 sec 2024-11-23T14:27:49,765 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees in 2.5750 sec 2024-11-23T14:27:49,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on 59fa5c31bb2297940cc7fa9d0ff623a3 2024-11-23T14:27:49,943 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T14:27:49,943 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 59fa5c31bb2297940cc7fa9d0ff623a3 3/3 column families, dataSize=174.43 KB heapSize=457.78 KB 2024-11-23T14:27:49,949 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123b7d6a2d7172347f5a7d618c8ddae90dc_59fa5c31bb2297940cc7fa9d0ff623a3 is 50, key is test_row_0/A:col10/1732372069942/Put/seqid=0 2024-11-23T14:27:49,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:49,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60448 deadline: 1732372129953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:49,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:49,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:49,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:49,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:49,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60476 deadline: 1732372129957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:49,961 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:49,961 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:49,961 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:49,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742459_1635 (size=14794) 2024-11-23T14:27:49,962 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:49,965 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123b7d6a2d7172347f5a7d618c8ddae90dc_59fa5c31bb2297940cc7fa9d0ff623a3 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123b7d6a2d7172347f5a7d618c8ddae90dc_59fa5c31bb2297940cc7fa9d0ff623a3 2024-11-23T14:27:49,965 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/A/447c0d6ae52e48189513d8d8b65d7fdd, store: [table=TestAcidGuarantees family=A region=59fa5c31bb2297940cc7fa9d0ff623a3] 2024-11-23T14:27:49,966 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/A/447c0d6ae52e48189513d8d8b65d7fdd is 175, key is test_row_0/A:col10/1732372069942/Put/seqid=0 2024-11-23T14:27:49,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742460_1636 (size=39749) 2024-11-23T14:27:50,062 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:50,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60448 deadline: 1732372130060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:50,062 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:50,062 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:50,062 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:50,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:50,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60476 deadline: 1732372130070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:50,072 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:50,072 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:50,072 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:50,110 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:50,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60470 deadline: 1732372130109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:50,111 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to c6f7b1897123,35385,1732371902767 for 'test_row_0' in 59fa5c31bb2297940cc7fa9d0ff623a3 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-23T14:27:50,111 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:50,111 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:50,111 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:50,122 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:50,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60492 deadline: 1732372130120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:50,122 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to c6f7b1897123,35385,1732371902767 for 'test_row_1' in 59fa5c31bb2297940cc7fa9d0ff623a3 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-23T14:27:50,123 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:50,123 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:50,123 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:50,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:50,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60468 deadline: 1732372130130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:50,133 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to c6f7b1897123,35385,1732371902767 for 'test_row_0' in 59fa5c31bb2297940cc7fa9d0ff623a3 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-23T14:27:50,133 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:50,133 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:50,133 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:50,272 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:50,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60448 deadline: 1732372130270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:50,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:50,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:50,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:50,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:50,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60476 deadline: 1732372130280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:50,282 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:50,282 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:50,282 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:50,370 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=243, memsize=58.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/A/447c0d6ae52e48189513d8d8b65d7fdd 2024-11-23T14:27:50,382 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/B/9de15cb0e78b4b979ed6e36b3a919b11 is 50, key is test_row_0/B:col10/1732372069942/Put/seqid=0 2024-11-23T14:27:50,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742461_1637 (size=12151) 2024-11-23T14:27:50,401 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/B/9de15cb0e78b4b979ed6e36b3a919b11 2024-11-23T14:27:50,407 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/C/ec2b9d7e50ee4d85a54717cd5c8e7dc9 is 50, key is test_row_0/C:col10/1732372069942/Put/seqid=0 2024-11-23T14:27:50,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742462_1638 (size=12151) 2024-11-23T14:27:50,411 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/C/ec2b9d7e50ee4d85a54717cd5c8e7dc9 2024-11-23T14:27:50,423 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/A/447c0d6ae52e48189513d8d8b65d7fdd as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/447c0d6ae52e48189513d8d8b65d7fdd 2024-11-23T14:27:50,427 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/447c0d6ae52e48189513d8d8b65d7fdd, entries=200, sequenceid=243, filesize=38.8 K 2024-11-23T14:27:50,428 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/B/9de15cb0e78b4b979ed6e36b3a919b11 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/9de15cb0e78b4b979ed6e36b3a919b11 2024-11-23T14:27:50,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,435 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/9de15cb0e78b4b979ed6e36b3a919b11, entries=150, sequenceid=243, filesize=11.9 K 2024-11-23T14:27:50,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,435 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/C/ec2b9d7e50ee4d85a54717cd5c8e7dc9 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/ec2b9d7e50ee4d85a54717cd5c8e7dc9 2024-11-23T14:27:50,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,438 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/ec2b9d7e50ee4d85a54717cd5c8e7dc9, entries=150, sequenceid=243, filesize=11.9 K 2024-11-23T14:27:50,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,438 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~457.73 KB/468720, currentSize=26.84 KB/27480 for 59fa5c31bb2297940cc7fa9d0ff623a3 in 495ms, sequenceid=243, compaction requested=true 2024-11-23T14:27:50,439 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 59fa5c31bb2297940cc7fa9d0ff623a3: 2024-11-23T14:27:50,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 59fa5c31bb2297940cc7fa9d0ff623a3:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T14:27:50,439 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:27:50,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:27:50,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 59fa5c31bb2297940cc7fa9d0ff623a3:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T14:27:50,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:27:50,439 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:27:50,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 59fa5c31bb2297940cc7fa9d0ff623a3:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T14:27:50,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:27:50,440 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:27:50,440 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102403 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:27:50,440 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1541): 59fa5c31bb2297940cc7fa9d0ff623a3/B is initiating minor compaction (all files) 2024-11-23T14:27:50,440 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1541): 59fa5c31bb2297940cc7fa9d0ff623a3/A is initiating minor compaction (all files) 2024-11-23T14:27:50,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,440 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 59fa5c31bb2297940cc7fa9d0ff623a3/A in TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:50,440 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 59fa5c31bb2297940cc7fa9d0ff623a3/B in TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:50,440 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/0c6163bf21d04f04baf227da10ebadd3, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/49f255242a6244e6a3ae655bfe087ed1, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/9de15cb0e78b4b979ed6e36b3a919b11] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp, totalSize=36.0 K 2024-11-23T14:27:50,440 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/66ff6aeeeb674cb29adac00f298dd2c9, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/0f325658aba0481286dd38abe30194cc, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/447c0d6ae52e48189513d8d8b65d7fdd] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp, totalSize=100.0 K 2024-11-23T14:27:50,440 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:50,440 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. files: [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/66ff6aeeeb674cb29adac00f298dd2c9, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/0f325658aba0481286dd38abe30194cc, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/447c0d6ae52e48189513d8d8b65d7fdd] 2024-11-23T14:27:50,440 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c6163bf21d04f04baf227da10ebadd3, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1732372065939 2024-11-23T14:27:50,440 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 66ff6aeeeb674cb29adac00f298dd2c9, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1732372065939 2024-11-23T14:27:50,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,440 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 49f255242a6244e6a3ae655bfe087ed1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732372067085 2024-11-23T14:27:50,441 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f325658aba0481286dd38abe30194cc, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732372067085 2024-11-23T14:27:50,441 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 9de15cb0e78b4b979ed6e36b3a919b11, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1732372069308 2024-11-23T14:27:50,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,441 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 447c0d6ae52e48189513d8d8b65d7fdd, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1732372069304 2024-11-23T14:27:50,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,447 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=59fa5c31bb2297940cc7fa9d0ff623a3] 2024-11-23T14:27:50,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,449 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 59fa5c31bb2297940cc7fa9d0ff623a3#B#compaction#540 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:27:50,449 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/B/e69c6aff3f1a4c8aa94ff1b38f2a7038 is 50, key is test_row_0/B:col10/1732372069942/Put/seqid=0 2024-11-23T14:27:50,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,450 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411231ff4469f1e71481f8f66f5934b3d95cb_59fa5c31bb2297940cc7fa9d0ff623a3 store=[table=TestAcidGuarantees family=A region=59fa5c31bb2297940cc7fa9d0ff623a3] 2024-11-23T14:27:50,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,452 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411231ff4469f1e71481f8f66f5934b3d95cb_59fa5c31bb2297940cc7fa9d0ff623a3, store=[table=TestAcidGuarantees family=A region=59fa5c31bb2297940cc7fa9d0ff623a3] 2024-11-23T14:27:50,452 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411231ff4469f1e71481f8f66f5934b3d95cb_59fa5c31bb2297940cc7fa9d0ff623a3 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=59fa5c31bb2297940cc7fa9d0ff623a3] 2024-11-23T14:27:50,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742463_1639 (size=12697) 2024-11-23T14:27:50,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,459 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/B/e69c6aff3f1a4c8aa94ff1b38f2a7038 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/e69c6aff3f1a4c8aa94ff1b38f2a7038 2024-11-23T14:27:50,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,467 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 59fa5c31bb2297940cc7fa9d0ff623a3/B of 59fa5c31bb2297940cc7fa9d0ff623a3 into e69c6aff3f1a4c8aa94ff1b38f2a7038(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:27:50,467 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 59fa5c31bb2297940cc7fa9d0ff623a3: 2024-11-23T14:27:50,467 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., storeName=59fa5c31bb2297940cc7fa9d0ff623a3/B, priority=13, startTime=1732372070439; duration=0sec 2024-11-23T14:27:50,467 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:27:50,467 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 59fa5c31bb2297940cc7fa9d0ff623a3:B 2024-11-23T14:27:50,467 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:27:50,469 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:27:50,469 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1541): 59fa5c31bb2297940cc7fa9d0ff623a3/C is initiating minor compaction (all files) 2024-11-23T14:27:50,469 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 59fa5c31bb2297940cc7fa9d0ff623a3/C in TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:50,469 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/c4d727d1f5124394829ddaccf4b8ee67, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/9f3e78c2bde44efc95dd7962ab74b55b, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/ec2b9d7e50ee4d85a54717cd5c8e7dc9] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp, totalSize=36.0 K 2024-11-23T14:27:50,469 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting c4d727d1f5124394829ddaccf4b8ee67, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1732372065939 2024-11-23T14:27:50,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742464_1640 (size=4469) 2024-11-23T14:27:50,470 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f3e78c2bde44efc95dd7962ab74b55b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732372067085 2024-11-23T14:27:50,471 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting ec2b9d7e50ee4d85a54717cd5c8e7dc9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1732372069308 2024-11-23T14:27:50,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,472 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 59fa5c31bb2297940cc7fa9d0ff623a3#A#compaction#539 average throughput is 0.98 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:27:50,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,473 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/A/8804ebd89cd74c8da5ca2a31b2381031 is 175, key is test_row_0/A:col10/1732372069942/Put/seqid=0 2024-11-23T14:27:50,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742465_1641 (size=31651) 2024-11-23T14:27:50,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,502 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 59fa5c31bb2297940cc7fa9d0ff623a3#C#compaction#541 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:27:50,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,502 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/C/158e090468ce41f0aa1ae34b30c85fb5 is 50, key is test_row_0/C:col10/1732372069942/Put/seqid=0 2024-11-23T14:27:50,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742466_1642 (size=12697) 2024-11-23T14:27:50,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on 59fa5c31bb2297940cc7fa9d0ff623a3 2024-11-23T14:27:50,594 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T14:27:50,594 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 59fa5c31bb2297940cc7fa9d0ff623a3 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-23T14:27:50,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,600 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411231b1b44e675f54fcf86f239711b07e282_59fa5c31bb2297940cc7fa9d0ff623a3 is 50, key is test_row_0/A:col10/1732372070593/Put/seqid=0 2024-11-23T14:27:50,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742467_1643 (size=19774) 2024-11-23T14:27:50,610 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:50,614 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411231b1b44e675f54fcf86f239711b07e282_59fa5c31bb2297940cc7fa9d0ff623a3 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411231b1b44e675f54fcf86f239711b07e282_59fa5c31bb2297940cc7fa9d0ff623a3 2024-11-23T14:27:50,615 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/A/378dd764800d4a679a410bab935c87ae, store: [table=TestAcidGuarantees family=A region=59fa5c31bb2297940cc7fa9d0ff623a3] 2024-11-23T14:27:50,615 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/A/378dd764800d4a679a410bab935c87ae is 175, key is test_row_0/A:col10/1732372070593/Put/seqid=0 2024-11-23T14:27:50,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742468_1644 (size=57033) 2024-11-23T14:27:50,638 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=255, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/A/378dd764800d4a679a410bab935c87ae 2024-11-23T14:27:50,652 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/B/b762fa7654ad4569bd82066db2d658b1 is 50, key is test_row_0/B:col10/1732372070593/Put/seqid=0 2024-11-23T14:27:50,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742469_1645 (size=12151) 2024-11-23T14:27:50,662 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/B/b762fa7654ad4569bd82066db2d658b1 2024-11-23T14:27:50,668 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:50,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60476 deadline: 1732372130664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:50,669 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:50,669 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:50,669 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:50,669 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:50,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60448 deadline: 1732372130664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:50,669 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/C/00208754d7194f979d90eedecad512a4 is 50, key is test_row_0/C:col10/1732372070593/Put/seqid=0 2024-11-23T14:27:50,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:50,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:50,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:50,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742470_1646 (size=12151) 2024-11-23T14:27:50,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:50,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60476 deadline: 1732372130780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:50,783 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:50,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:50,783 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:50,783 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:50,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60448 deadline: 1732372130780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:50,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:50,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:50,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:50,888 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/A/8804ebd89cd74c8da5ca2a31b2381031 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/8804ebd89cd74c8da5ca2a31b2381031 2024-11-23T14:27:50,892 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 59fa5c31bb2297940cc7fa9d0ff623a3/A of 59fa5c31bb2297940cc7fa9d0ff623a3 into 8804ebd89cd74c8da5ca2a31b2381031(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:27:50,892 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 59fa5c31bb2297940cc7fa9d0ff623a3: 2024-11-23T14:27:50,892 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., storeName=59fa5c31bb2297940cc7fa9d0ff623a3/A, priority=13, startTime=1732372070439; duration=0sec 2024-11-23T14:27:50,892 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:27:50,892 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 59fa5c31bb2297940cc7fa9d0ff623a3:A 2024-11-23T14:27:50,911 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/C/158e090468ce41f0aa1ae34b30c85fb5 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/158e090468ce41f0aa1ae34b30c85fb5 2024-11-23T14:27:50,914 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 59fa5c31bb2297940cc7fa9d0ff623a3/C of 59fa5c31bb2297940cc7fa9d0ff623a3 into 158e090468ce41f0aa1ae34b30c85fb5(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:27:50,914 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 59fa5c31bb2297940cc7fa9d0ff623a3: 2024-11-23T14:27:50,914 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., storeName=59fa5c31bb2297940cc7fa9d0ff623a3/C, priority=13, startTime=1732372070439; duration=0sec 2024-11-23T14:27:50,914 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:27:50,914 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 59fa5c31bb2297940cc7fa9d0ff623a3:C 2024-11-23T14:27:50,992 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:50,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60476 deadline: 1732372130990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:50,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:50,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60448 deadline: 1732372130990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:50,992 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:50,992 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:50,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:50,992 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:50,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:50,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:51,072 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/C/00208754d7194f979d90eedecad512a4 2024-11-23T14:27:51,075 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/A/378dd764800d4a679a410bab935c87ae as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/378dd764800d4a679a410bab935c87ae 2024-11-23T14:27:51,078 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/378dd764800d4a679a410bab935c87ae, entries=300, sequenceid=255, filesize=55.7 K 2024-11-23T14:27:51,079 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/B/b762fa7654ad4569bd82066db2d658b1 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/b762fa7654ad4569bd82066db2d658b1 2024-11-23T14:27:51,081 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/b762fa7654ad4569bd82066db2d658b1, entries=150, sequenceid=255, filesize=11.9 K 2024-11-23T14:27:51,082 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/C/00208754d7194f979d90eedecad512a4 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/00208754d7194f979d90eedecad512a4 2024-11-23T14:27:51,084 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/00208754d7194f979d90eedecad512a4, entries=150, sequenceid=255, filesize=11.9 K 2024-11-23T14:27:51,085 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 59fa5c31bb2297940cc7fa9d0ff623a3 in 491ms, sequenceid=255, compaction requested=false 2024-11-23T14:27:51,085 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 59fa5c31bb2297940cc7fa9d0ff623a3: 2024-11-23T14:27:51,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on 59fa5c31bb2297940cc7fa9d0ff623a3 2024-11-23T14:27:51,304 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T14:27:51,304 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 59fa5c31bb2297940cc7fa9d0ff623a3 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-23T14:27:51,310 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123c28c2ff191f14201baf85924f5caf1c2_59fa5c31bb2297940cc7fa9d0ff623a3 is 50, key is test_row_0/A:col10/1732372070640/Put/seqid=0 2024-11-23T14:27:51,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742471_1647 (size=17534) 2024-11-23T14:27:51,317 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:51,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60448 deadline: 1732372131312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:51,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:51,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:51,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:51,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:51,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60476 deadline: 1732372131316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:51,318 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:51,318 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:51,318 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:51,324 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:51,326 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123c28c2ff191f14201baf85924f5caf1c2_59fa5c31bb2297940cc7fa9d0ff623a3 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123c28c2ff191f14201baf85924f5caf1c2_59fa5c31bb2297940cc7fa9d0ff623a3 2024-11-23T14:27:51,327 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/A/bde9752367774277a70ef6d28dbe7666, store: [table=TestAcidGuarantees family=A region=59fa5c31bb2297940cc7fa9d0ff623a3] 2024-11-23T14:27:51,328 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/A/bde9752367774277a70ef6d28dbe7666 is 175, key is test_row_0/A:col10/1732372070640/Put/seqid=0 2024-11-23T14:27:51,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742472_1648 (size=48639) 2024-11-23T14:27:51,336 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=285, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/A/bde9752367774277a70ef6d28dbe7666 2024-11-23T14:27:51,342 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/B/9c53898f74fe45d1aecbf01cf642e6d0 is 50, key is test_row_0/B:col10/1732372070640/Put/seqid=0 2024-11-23T14:27:51,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742473_1649 (size=12301) 2024-11-23T14:27:51,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-23T14:27:51,360 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T14:27:51,361 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-23T14:27:51,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees 2024-11-23T14:27:51,362 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T14:27:51,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-23T14:27:51,363 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T14:27:51,363 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T14:27:51,421 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:51,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60448 deadline: 1732372131420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:51,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:51,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:51,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:51,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:51,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60476 deadline: 1732372131420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:51,422 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:51,422 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:51,422 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:51,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-23T14:27:51,516 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-23T14:27:51,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:51,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. as already flushing 2024-11-23T14:27:51,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:51,516 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:51,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:51,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:51,633 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:51,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60448 deadline: 1732372131629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:51,633 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:51,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:51,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60476 deadline: 1732372131630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:51,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:51,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:51,634 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:51,634 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:51,634 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:51,668 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-23T14:27:51,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:51,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. as already flushing 2024-11-23T14:27:51,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:51,668 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:51,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:51,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:51,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-23T14:27:51,747 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/B/9c53898f74fe45d1aecbf01cf642e6d0 2024-11-23T14:27:51,753 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/C/5bb6e0acbb1a443c9936990a30e8490e is 50, key is test_row_0/C:col10/1732372070640/Put/seqid=0 2024-11-23T14:27:51,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742474_1650 (size=12301) 2024-11-23T14:27:51,821 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-23T14:27:51,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:51,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. as already flushing 2024-11-23T14:27:51,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:51,821 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:51,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:51,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:51,942 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:51,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60448 deadline: 1732372131940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:51,942 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:51,942 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:51,942 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:51,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60476 deadline: 1732372131940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:51,942 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:51,942 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:51,943 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:51,943 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:51,975 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-23T14:27:51,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:51,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. as already flushing 2024-11-23T14:27:51,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:51,975 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:51,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:51,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:51,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-23T14:27:52,128 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-23T14:27:52,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:52,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. as already flushing 2024-11-23T14:27:52,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:52,128 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:52,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:52,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:52,171 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/C/5bb6e0acbb1a443c9936990a30e8490e 2024-11-23T14:27:52,176 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/A/bde9752367774277a70ef6d28dbe7666 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/bde9752367774277a70ef6d28dbe7666 2024-11-23T14:27:52,179 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/bde9752367774277a70ef6d28dbe7666, entries=250, sequenceid=285, filesize=47.5 K 2024-11-23T14:27:52,180 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/B/9c53898f74fe45d1aecbf01cf642e6d0 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/9c53898f74fe45d1aecbf01cf642e6d0 2024-11-23T14:27:52,189 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/9c53898f74fe45d1aecbf01cf642e6d0, entries=150, sequenceid=285, filesize=12.0 K 2024-11-23T14:27:52,190 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/C/5bb6e0acbb1a443c9936990a30e8490e as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/5bb6e0acbb1a443c9936990a30e8490e 2024-11-23T14:27:52,193 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/5bb6e0acbb1a443c9936990a30e8490e, entries=150, sequenceid=285, filesize=12.0 K 2024-11-23T14:27:52,194 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for 59fa5c31bb2297940cc7fa9d0ff623a3 in 890ms, sequenceid=285, compaction requested=true 2024-11-23T14:27:52,194 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 59fa5c31bb2297940cc7fa9d0ff623a3: 2024-11-23T14:27:52,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 59fa5c31bb2297940cc7fa9d0ff623a3:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T14:27:52,194 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:27:52,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:27:52,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 59fa5c31bb2297940cc7fa9d0ff623a3:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T14:27:52,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:27:52,194 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:27:52,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 59fa5c31bb2297940cc7fa9d0ff623a3:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T14:27:52,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:27:52,194 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 137323 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:27:52,194 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:27:52,195 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1541): 59fa5c31bb2297940cc7fa9d0ff623a3/B is initiating minor compaction (all files) 2024-11-23T14:27:52,195 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1541): 59fa5c31bb2297940cc7fa9d0ff623a3/A is initiating minor compaction (all files) 2024-11-23T14:27:52,195 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 59fa5c31bb2297940cc7fa9d0ff623a3/A in TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:52,195 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 59fa5c31bb2297940cc7fa9d0ff623a3/B in TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:52,195 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/8804ebd89cd74c8da5ca2a31b2381031, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/378dd764800d4a679a410bab935c87ae, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/bde9752367774277a70ef6d28dbe7666] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp, totalSize=134.1 K 2024-11-23T14:27:52,195 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/e69c6aff3f1a4c8aa94ff1b38f2a7038, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/b762fa7654ad4569bd82066db2d658b1, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/9c53898f74fe45d1aecbf01cf642e6d0] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp, totalSize=36.3 K 2024-11-23T14:27:52,195 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:52,195 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. files: [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/8804ebd89cd74c8da5ca2a31b2381031, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/378dd764800d4a679a410bab935c87ae, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/bde9752367774277a70ef6d28dbe7666] 2024-11-23T14:27:52,195 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting e69c6aff3f1a4c8aa94ff1b38f2a7038, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1732372069308 2024-11-23T14:27:52,195 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8804ebd89cd74c8da5ca2a31b2381031, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1732372069308 2024-11-23T14:27:52,195 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting b762fa7654ad4569bd82066db2d658b1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1732372070582 2024-11-23T14:27:52,195 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 378dd764800d4a679a410bab935c87ae, keycount=300, bloomtype=ROW, size=55.7 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1732372069951 2024-11-23T14:27:52,196 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting bde9752367774277a70ef6d28dbe7666, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1732372070635 2024-11-23T14:27:52,196 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c53898f74fe45d1aecbf01cf642e6d0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1732372070640 2024-11-23T14:27:52,200 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=59fa5c31bb2297940cc7fa9d0ff623a3] 2024-11-23T14:27:52,206 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 59fa5c31bb2297940cc7fa9d0ff623a3#B#compaction#549 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:27:52,206 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/B/6171f780d0ed44b3b76aad1993631682 is 50, key is test_row_0/B:col10/1732372070640/Put/seqid=0 2024-11-23T14:27:52,212 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241123c8623b7240df4c069149858304653d03_59fa5c31bb2297940cc7fa9d0ff623a3 store=[table=TestAcidGuarantees family=A region=59fa5c31bb2297940cc7fa9d0ff623a3] 2024-11-23T14:27:52,214 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241123c8623b7240df4c069149858304653d03_59fa5c31bb2297940cc7fa9d0ff623a3, store=[table=TestAcidGuarantees family=A region=59fa5c31bb2297940cc7fa9d0ff623a3] 2024-11-23T14:27:52,214 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123c8623b7240df4c069149858304653d03_59fa5c31bb2297940cc7fa9d0ff623a3 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=59fa5c31bb2297940cc7fa9d0ff623a3] 2024-11-23T14:27:52,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742475_1651 (size=12949) 2024-11-23T14:27:52,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742476_1652 (size=4469) 2024-11-23T14:27:52,228 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 59fa5c31bb2297940cc7fa9d0ff623a3#A#compaction#548 average throughput is 0.87 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:27:52,229 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/A/10533be2358a4af9b99b09aecbcdde04 is 175, key is test_row_0/A:col10/1732372070640/Put/seqid=0 2024-11-23T14:27:52,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742477_1653 (size=31903) 2024-11-23T14:27:52,236 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/A/10533be2358a4af9b99b09aecbcdde04 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/10533be2358a4af9b99b09aecbcdde04 2024-11-23T14:27:52,240 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 59fa5c31bb2297940cc7fa9d0ff623a3/A of 59fa5c31bb2297940cc7fa9d0ff623a3 into 10533be2358a4af9b99b09aecbcdde04(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:27:52,240 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 59fa5c31bb2297940cc7fa9d0ff623a3: 2024-11-23T14:27:52,240 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., storeName=59fa5c31bb2297940cc7fa9d0ff623a3/A, priority=13, startTime=1732372072194; duration=0sec 2024-11-23T14:27:52,240 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:27:52,240 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 59fa5c31bb2297940cc7fa9d0ff623a3:A 2024-11-23T14:27:52,240 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:27:52,241 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:27:52,241 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1541): 59fa5c31bb2297940cc7fa9d0ff623a3/C is initiating minor compaction (all files) 2024-11-23T14:27:52,241 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 59fa5c31bb2297940cc7fa9d0ff623a3/C in TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:52,241 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/158e090468ce41f0aa1ae34b30c85fb5, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/00208754d7194f979d90eedecad512a4, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/5bb6e0acbb1a443c9936990a30e8490e] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp, totalSize=36.3 K 2024-11-23T14:27:52,241 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 158e090468ce41f0aa1ae34b30c85fb5, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1732372069308 2024-11-23T14:27:52,241 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 00208754d7194f979d90eedecad512a4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1732372070582 2024-11-23T14:27:52,242 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5bb6e0acbb1a443c9936990a30e8490e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1732372070640 2024-11-23T14:27:52,250 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 59fa5c31bb2297940cc7fa9d0ff623a3#C#compaction#550 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:27:52,251 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/C/cc34ce52d7934a4fa7bcb59213027677 is 50, key is test_row_0/C:col10/1732372070640/Put/seqid=0 2024-11-23T14:27:52,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742478_1654 (size=12949) 2024-11-23T14:27:52,275 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/C/cc34ce52d7934a4fa7bcb59213027677 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/cc34ce52d7934a4fa7bcb59213027677 2024-11-23T14:27:52,280 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-23T14:27:52,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:52,280 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2902): Flushing 59fa5c31bb2297940cc7fa9d0ff623a3 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-23T14:27:52,281 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 59fa5c31bb2297940cc7fa9d0ff623a3/C of 59fa5c31bb2297940cc7fa9d0ff623a3 into cc34ce52d7934a4fa7bcb59213027677(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:27:52,281 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 59fa5c31bb2297940cc7fa9d0ff623a3: 2024-11-23T14:27:52,281 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., storeName=59fa5c31bb2297940cc7fa9d0ff623a3/C, priority=13, startTime=1732372072194; duration=0sec 2024-11-23T14:27:52,281 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:27:52,281 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 59fa5c31bb2297940cc7fa9d0ff623a3:C 2024-11-23T14:27:52,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411236fd0fad327cd4de2a37a5ceef7e706d3_59fa5c31bb2297940cc7fa9d0ff623a3 is 50, key is test_row_0/A:col10/1732372071314/Put/seqid=0 2024-11-23T14:27:52,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742479_1655 (size=12454) 2024-11-23T14:27:52,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:52,331 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411236fd0fad327cd4de2a37a5ceef7e706d3_59fa5c31bb2297940cc7fa9d0ff623a3 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411236fd0fad327cd4de2a37a5ceef7e706d3_59fa5c31bb2297940cc7fa9d0ff623a3 2024-11-23T14:27:52,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/A/6902365158414bb29509a2f2a1e4e5b9, store: [table=TestAcidGuarantees family=A region=59fa5c31bb2297940cc7fa9d0ff623a3] 2024-11-23T14:27:52,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/A/6902365158414bb29509a2f2a1e4e5b9 is 175, key is test_row_0/A:col10/1732372071314/Put/seqid=0 2024-11-23T14:27:52,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742480_1656 (size=31255) 2024-11-23T14:27:52,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on 59fa5c31bb2297940cc7fa9d0ff623a3 2024-11-23T14:27:52,457 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. as already flushing 2024-11-23T14:27:52,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-23T14:27:52,501 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:52,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60476 deadline: 1732372132499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:52,502 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:52,502 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:52,502 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:52,504 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:52,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60448 deadline: 1732372132502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:52,505 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:52,505 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:52,505 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:52,611 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:52,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60476 deadline: 1732372132610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:52,612 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:52,612 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:52,612 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:52,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60448 deadline: 1732372132610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:52,612 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:52,612 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:52,612 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:52,612 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:52,629 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/B/6171f780d0ed44b3b76aad1993631682 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/6171f780d0ed44b3b76aad1993631682 2024-11-23T14:27:52,633 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 59fa5c31bb2297940cc7fa9d0ff623a3/B of 59fa5c31bb2297940cc7fa9d0ff623a3 into 6171f780d0ed44b3b76aad1993631682(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:27:52,633 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 59fa5c31bb2297940cc7fa9d0ff623a3: 2024-11-23T14:27:52,633 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., storeName=59fa5c31bb2297940cc7fa9d0ff623a3/B, priority=13, startTime=1732372072194; duration=0sec 2024-11-23T14:27:52,633 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:27:52,633 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 59fa5c31bb2297940cc7fa9d0ff623a3:B 2024-11-23T14:27:52,743 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=295, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/A/6902365158414bb29509a2f2a1e4e5b9 2024-11-23T14:27:52,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/B/c42f4b5501414add990c4c7c67f8cffd is 50, key is test_row_0/B:col10/1732372071314/Put/seqid=0 2024-11-23T14:27:52,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742481_1657 (size=12301) 2024-11-23T14:27:52,752 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/B/c42f4b5501414add990c4c7c67f8cffd 2024-11-23T14:27:52,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/C/1c9896c1f94a4fb6aadaac2fc0fe2344 is 50, key is test_row_0/C:col10/1732372071314/Put/seqid=0 2024-11-23T14:27:52,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742482_1658 (size=12301) 2024-11-23T14:27:52,761 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/C/1c9896c1f94a4fb6aadaac2fc0fe2344 2024-11-23T14:27:52,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/A/6902365158414bb29509a2f2a1e4e5b9 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/6902365158414bb29509a2f2a1e4e5b9 2024-11-23T14:27:52,783 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/6902365158414bb29509a2f2a1e4e5b9, entries=150, sequenceid=295, filesize=30.5 K 2024-11-23T14:27:52,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/B/c42f4b5501414add990c4c7c67f8cffd as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/c42f4b5501414add990c4c7c67f8cffd 2024-11-23T14:27:52,788 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/c42f4b5501414add990c4c7c67f8cffd, entries=150, sequenceid=295, filesize=12.0 K 2024-11-23T14:27:52,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/C/1c9896c1f94a4fb6aadaac2fc0fe2344 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/1c9896c1f94a4fb6aadaac2fc0fe2344 2024-11-23T14:27:52,794 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/1c9896c1f94a4fb6aadaac2fc0fe2344, entries=150, sequenceid=295, filesize=12.0 K 2024-11-23T14:27:52,795 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=167.72 KB/171750 for 59fa5c31bb2297940cc7fa9d0ff623a3 in 515ms, sequenceid=295, compaction requested=false 2024-11-23T14:27:52,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2603): Flush status journal for 59fa5c31bb2297940cc7fa9d0ff623a3: 2024-11-23T14:27:52,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:52,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=167 2024-11-23T14:27:52,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster(4169): Remote procedure done, pid=167 2024-11-23T14:27:52,800 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-11-23T14:27:52,800 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4340 sec 2024-11-23T14:27:52,802 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees in 1.4390 sec 2024-11-23T14:27:52,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on 59fa5c31bb2297940cc7fa9d0ff623a3 2024-11-23T14:27:52,823 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T14:27:52,823 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 59fa5c31bb2297940cc7fa9d0ff623a3 3/3 column families, dataSize=174.43 KB heapSize=457.78 KB 2024-11-23T14:27:52,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:52,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60476 deadline: 1732372132830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:52,832 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:52,832 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:52,832 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:52,834 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:52,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60448 deadline: 1732372132831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:52,834 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:52,834 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:52,834 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:52,837 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123003a5e59486143189aec5ca878d837c0_59fa5c31bb2297940cc7fa9d0ff623a3 is 50, key is test_row_0/A:col10/1732372072497/Put/seqid=0 2024-11-23T14:27:52,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742483_1659 (size=14994) 2024-11-23T14:27:52,940 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:52,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60476 deadline: 1732372132940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:52,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:52,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60448 deadline: 1732372132940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:52,941 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:52,941 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:52,941 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:52,941 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:52,941 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:52,941 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:53,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:53,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60476 deadline: 1732372133149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:53,152 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:53,152 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:53,152 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:53,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:53,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60448 deadline: 1732372133150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:53,154 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:53,154 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:53,154 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:53,250 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,253 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123003a5e59486143189aec5ca878d837c0_59fa5c31bb2297940cc7fa9d0ff623a3 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123003a5e59486143189aec5ca878d837c0_59fa5c31bb2297940cc7fa9d0ff623a3 2024-11-23T14:27:53,253 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/A/03593d1c5a3c42e6aa86693dde102fe0, store: [table=TestAcidGuarantees family=A region=59fa5c31bb2297940cc7fa9d0ff623a3] 2024-11-23T14:27:53,254 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/A/03593d1c5a3c42e6aa86693dde102fe0 is 175, key is test_row_0/A:col10/1732372072497/Put/seqid=0 2024-11-23T14:27:53,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742484_1660 (size=39949) 2024-11-23T14:27:53,257 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=326, memsize=60.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/A/03593d1c5a3c42e6aa86693dde102fe0 2024-11-23T14:27:53,266 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/B/6239373ad90c451eb34c7fd51fb48300 is 50, key is test_row_0/B:col10/1732372072497/Put/seqid=0 2024-11-23T14:27:53,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742485_1661 (size=12301) 2024-11-23T14:27:53,462 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:53,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60476 deadline: 1732372133460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:53,462 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:53,462 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:53,462 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:53,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60448 deadline: 1732372133460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:53,462 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:53,462 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:53,462 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:53,462 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:53,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-23T14:27:53,510 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T14:27:53,510 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-23T14:27:53,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees 2024-11-23T14:27:53,511 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T14:27:53,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-23T14:27:53,512 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T14:27:53,512 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T14:27:53,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-23T14:27:53,664 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-23T14:27:53,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:53,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. as already flushing 2024-11-23T14:27:53,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:53,664 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:53,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:53,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:53,690 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/B/6239373ad90c451eb34c7fd51fb48300 2024-11-23T14:27:53,695 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/C/a0047a773c7d4edbaf2c09476628e467 is 50, key is test_row_0/C:col10/1732372072497/Put/seqid=0 2024-11-23T14:27:53,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742486_1662 (size=12301) 2024-11-23T14:27:53,699 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/C/a0047a773c7d4edbaf2c09476628e467 2024-11-23T14:27:53,702 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/A/03593d1c5a3c42e6aa86693dde102fe0 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/03593d1c5a3c42e6aa86693dde102fe0 2024-11-23T14:27:53,704 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/03593d1c5a3c42e6aa86693dde102fe0, entries=200, sequenceid=326, filesize=39.0 K 2024-11-23T14:27:53,705 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/B/6239373ad90c451eb34c7fd51fb48300 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/6239373ad90c451eb34c7fd51fb48300 2024-11-23T14:27:53,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,708 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/6239373ad90c451eb34c7fd51fb48300, entries=150, sequenceid=326, filesize=12.0 K 2024-11-23T14:27:53,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,709 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/C/a0047a773c7d4edbaf2c09476628e467 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/a0047a773c7d4edbaf2c09476628e467 2024-11-23T14:27:53,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,711 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/a0047a773c7d4edbaf2c09476628e467, entries=150, sequenceid=326, filesize=12.0 K 2024-11-23T14:27:53,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,712 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~181.14 KB/185490, heapSize ~475.31 KB/486720, currentSize=20.13 KB/20610 for 59fa5c31bb2297940cc7fa9d0ff623a3 in 889ms, sequenceid=326, compaction requested=true 2024-11-23T14:27:53,712 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 59fa5c31bb2297940cc7fa9d0ff623a3: 2024-11-23T14:27:53,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 59fa5c31bb2297940cc7fa9d0ff623a3:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T14:27:53,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:27:53,712 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:27:53,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 59fa5c31bb2297940cc7fa9d0ff623a3:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T14:27:53,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:27:53,712 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:27:53,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 59fa5c31bb2297940cc7fa9d0ff623a3:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T14:27:53,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:27:53,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,713 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103107 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:27:53,713 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1541): 59fa5c31bb2297940cc7fa9d0ff623a3/A is initiating minor compaction (all files) 2024-11-23T14:27:53,713 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 59fa5c31bb2297940cc7fa9d0ff623a3/A in TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:53,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,713 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/10533be2358a4af9b99b09aecbcdde04, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/6902365158414bb29509a2f2a1e4e5b9, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/03593d1c5a3c42e6aa86693dde102fe0] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp, totalSize=100.7 K 2024-11-23T14:27:53,713 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:53,713 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. files: [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/10533be2358a4af9b99b09aecbcdde04, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/6902365158414bb29509a2f2a1e4e5b9, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/03593d1c5a3c42e6aa86693dde102fe0] 2024-11-23T14:27:53,714 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 10533be2358a4af9b99b09aecbcdde04, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1732372070640 2024-11-23T14:27:53,714 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:27:53,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,714 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1541): 59fa5c31bb2297940cc7fa9d0ff623a3/B is initiating minor compaction (all files) 2024-11-23T14:27:53,714 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 59fa5c31bb2297940cc7fa9d0ff623a3/B in TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:53,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,714 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/6171f780d0ed44b3b76aad1993631682, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/c42f4b5501414add990c4c7c67f8cffd, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/6239373ad90c451eb34c7fd51fb48300] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp, totalSize=36.7 K 2024-11-23T14:27:53,714 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6902365158414bb29509a2f2a1e4e5b9, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1732372071307 2024-11-23T14:27:53,714 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 03593d1c5a3c42e6aa86693dde102fe0, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1732372072497 2024-11-23T14:27:53,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,714 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 6171f780d0ed44b3b76aad1993631682, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1732372070640 2024-11-23T14:27:53,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,715 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting c42f4b5501414add990c4c7c67f8cffd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1732372071307 2024-11-23T14:27:53,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,716 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 6239373ad90c451eb34c7fd51fb48300, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1732372072497 2024-11-23T14:27:53,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,729 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=59fa5c31bb2297940cc7fa9d0ff623a3] 2024-11-23T14:27:53,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,730 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 59fa5c31bb2297940cc7fa9d0ff623a3#B#compaction#557 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:27:53,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,730 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/B/b40dbbbdc6f643878145c88bd9a3331e is 50, key is test_row_0/B:col10/1732372072497/Put/seqid=0 2024-11-23T14:27:53,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,745 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241123516ddb4a32ee42b9891373147864e74e_59fa5c31bb2297940cc7fa9d0ff623a3 store=[table=TestAcidGuarantees family=A region=59fa5c31bb2297940cc7fa9d0ff623a3] 2024-11-23T14:27:53,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,746 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241123516ddb4a32ee42b9891373147864e74e_59fa5c31bb2297940cc7fa9d0ff623a3, store=[table=TestAcidGuarantees family=A region=59fa5c31bb2297940cc7fa9d0ff623a3] 2024-11-23T14:27:53,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,747 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123516ddb4a32ee42b9891373147864e74e_59fa5c31bb2297940cc7fa9d0ff623a3 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=59fa5c31bb2297940cc7fa9d0ff623a3] 2024-11-23T14:27:53,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742487_1663 (size=13051) 2024-11-23T14:27:53,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742488_1664 (size=4469) 2024-11-23T14:27:53,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,766 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 59fa5c31bb2297940cc7fa9d0ff623a3#A#compaction#558 average throughput is 0.66 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:27:53,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,766 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/A/3cf17acce4904588aedc51762e587565 is 175, key is test_row_0/A:col10/1732372072497/Put/seqid=0 2024-11-23T14:27:53,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742489_1665 (size=32005) 2024-11-23T14:27:53,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,816 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-23T14:27:53,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:53,817 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2902): Flushing 59fa5c31bb2297940cc7fa9d0ff623a3 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-11-23T14:27:53,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411230078cb74fdb047c2aceae71a871a5081_59fa5c31bb2297940cc7fa9d0ff623a3 is 50, key is test_row_1/A:col10/1732372072828/Put/seqid=0 2024-11-23T14:27:53,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742490_1666 (size=9914) 2024-11-23T14:27:53,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-23T14:27:53,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on 59fa5c31bb2297940cc7fa9d0ff623a3 2024-11-23T14:27:53,993 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. as already flushing 2024-11-23T14:27:53,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:53,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:54,051 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:54,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60476 deadline: 1732372134050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:54,052 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:54,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:54,052 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:54,052 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:54,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60448 deadline: 1732372134052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:54,052 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:54,052 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:54,052 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:54,136 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:54,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60470 deadline: 1732372134136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:54,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-23T14:27:54,141 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to c6f7b1897123,35385,1732371902767 for 'test_row_0' in 59fa5c31bb2297940cc7fa9d0ff623a3 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-23T14:27:54,141 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:54,141 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:54,141 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:54,143 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:54,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60492 deadline: 1732372134141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:54,144 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to c6f7b1897123,35385,1732371902767 for 'test_row_1' in 59fa5c31bb2297940cc7fa9d0ff623a3 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-23T14:27:54,144 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:54,144 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:54,145 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:54,153 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:54,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60468 deadline: 1732372134150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:54,153 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/B/b40dbbbdc6f643878145c88bd9a3331e as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/b40dbbbdc6f643878145c88bd9a3331e 2024-11-23T14:27:54,153 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to c6f7b1897123,35385,1732371902767 for 'test_row_0' in 59fa5c31bb2297940cc7fa9d0ff623a3 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-23T14:27:54,154 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:54,154 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:54,154 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:54,157 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 59fa5c31bb2297940cc7fa9d0ff623a3/B of 59fa5c31bb2297940cc7fa9d0ff623a3 into b40dbbbdc6f643878145c88bd9a3331e(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:27:54,157 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 59fa5c31bb2297940cc7fa9d0ff623a3: 2024-11-23T14:27:54,157 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., storeName=59fa5c31bb2297940cc7fa9d0ff623a3/B, priority=13, startTime=1732372073712; duration=0sec 2024-11-23T14:27:54,157 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:27:54,157 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 59fa5c31bb2297940cc7fa9d0ff623a3:B 2024-11-23T14:27:54,157 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:27:54,158 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:27:54,158 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1541): 59fa5c31bb2297940cc7fa9d0ff623a3/C is initiating minor compaction (all files) 2024-11-23T14:27:54,158 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 59fa5c31bb2297940cc7fa9d0ff623a3/C in TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:54,158 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/cc34ce52d7934a4fa7bcb59213027677, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/1c9896c1f94a4fb6aadaac2fc0fe2344, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/a0047a773c7d4edbaf2c09476628e467] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp, totalSize=36.7 K 2024-11-23T14:27:54,158 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting cc34ce52d7934a4fa7bcb59213027677, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1732372070640 2024-11-23T14:27:54,158 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c9896c1f94a4fb6aadaac2fc0fe2344, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1732372071307 2024-11-23T14:27:54,158 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting a0047a773c7d4edbaf2c09476628e467, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1732372072497 2024-11-23T14:27:54,162 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:54,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60476 deadline: 1732372134160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:54,162 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:54,162 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:54,162 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:54,163 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:54,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60448 deadline: 1732372134160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:54,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:54,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:54,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:54,164 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 59fa5c31bb2297940cc7fa9d0ff623a3#C#compaction#560 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:27:54,164 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/C/96c5967fe737417ab1a40136c3dac858 is 50, key is test_row_0/C:col10/1732372072497/Put/seqid=0 2024-11-23T14:27:54,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742491_1667 (size=13051) 2024-11-23T14:27:54,185 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/A/3cf17acce4904588aedc51762e587565 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/3cf17acce4904588aedc51762e587565 2024-11-23T14:27:54,189 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 59fa5c31bb2297940cc7fa9d0ff623a3/A of 59fa5c31bb2297940cc7fa9d0ff623a3 into 3cf17acce4904588aedc51762e587565(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:27:54,189 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 59fa5c31bb2297940cc7fa9d0ff623a3: 2024-11-23T14:27:54,189 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., storeName=59fa5c31bb2297940cc7fa9d0ff623a3/A, priority=13, startTime=1732372073712; duration=0sec 2024-11-23T14:27:54,189 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:27:54,189 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 59fa5c31bb2297940cc7fa9d0ff623a3:A 2024-11-23T14:27:54,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:54,232 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411230078cb74fdb047c2aceae71a871a5081_59fa5c31bb2297940cc7fa9d0ff623a3 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411230078cb74fdb047c2aceae71a871a5081_59fa5c31bb2297940cc7fa9d0ff623a3 2024-11-23T14:27:54,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/A/207c67522e7044d780f98a713df8af92, store: [table=TestAcidGuarantees family=A region=59fa5c31bb2297940cc7fa9d0ff623a3] 2024-11-23T14:27:54,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/A/207c67522e7044d780f98a713df8af92 is 175, key is test_row_1/A:col10/1732372072828/Put/seqid=0 2024-11-23T14:27:54,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742492_1668 (size=22561) 2024-11-23T14:27:54,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:54,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60476 deadline: 1732372134370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:54,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:54,373 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:54,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60448 deadline: 1732372134370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:54,373 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:54,373 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:54,373 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:54,373 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:54,373 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:54,570 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/C/96c5967fe737417ab1a40136c3dac858 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/96c5967fe737417ab1a40136c3dac858 2024-11-23T14:27:54,573 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 59fa5c31bb2297940cc7fa9d0ff623a3/C of 59fa5c31bb2297940cc7fa9d0ff623a3 into 96c5967fe737417ab1a40136c3dac858(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:27:54,573 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 59fa5c31bb2297940cc7fa9d0ff623a3: 2024-11-23T14:27:54,574 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., storeName=59fa5c31bb2297940cc7fa9d0ff623a3/C, priority=13, startTime=1732372073712; duration=0sec 2024-11-23T14:27:54,574 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:27:54,574 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 59fa5c31bb2297940cc7fa9d0ff623a3:C 2024-11-23T14:27:54,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-23T14:27:54,650 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=332, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/A/207c67522e7044d780f98a713df8af92 2024-11-23T14:27:54,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/B/7152ce9a0c314ab58a343ca7ea57ccc3 is 50, key is test_row_1/B:col10/1732372072828/Put/seqid=0 2024-11-23T14:27:54,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742493_1669 (size=9857) 2024-11-23T14:27:54,683 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:54,683 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:54,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60476 deadline: 1732372134680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:54,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60448 deadline: 1732372134680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:54,683 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:54,683 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:54,683 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:54,684 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:54,684 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:54,684 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:55,061 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/B/7152ce9a0c314ab58a343ca7ea57ccc3 2024-11-23T14:27:55,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/C/6468e2c4e7be44059583ea36ee17c7a1 is 50, key is test_row_1/C:col10/1732372072828/Put/seqid=0 2024-11-23T14:27:55,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742494_1670 (size=9857) 2024-11-23T14:27:55,068 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/C/6468e2c4e7be44059583ea36ee17c7a1 2024-11-23T14:27:55,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/A/207c67522e7044d780f98a713df8af92 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/207c67522e7044d780f98a713df8af92 2024-11-23T14:27:55,074 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/207c67522e7044d780f98a713df8af92, entries=100, sequenceid=332, filesize=22.0 K 2024-11-23T14:27:55,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/B/7152ce9a0c314ab58a343ca7ea57ccc3 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/7152ce9a0c314ab58a343ca7ea57ccc3 2024-11-23T14:27:55,077 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/7152ce9a0c314ab58a343ca7ea57ccc3, entries=100, sequenceid=332, filesize=9.6 K 2024-11-23T14:27:55,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/C/6468e2c4e7be44059583ea36ee17c7a1 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/6468e2c4e7be44059583ea36ee17c7a1 2024-11-23T14:27:55,080 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/6468e2c4e7be44059583ea36ee17c7a1, entries=100, sequenceid=332, filesize=9.6 K 2024-11-23T14:27:55,081 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=181.14 KB/185490 for 59fa5c31bb2297940cc7fa9d0ff623a3 in 1264ms, sequenceid=332, compaction requested=false 2024-11-23T14:27:55,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2603): Flush status journal for 59fa5c31bb2297940cc7fa9d0ff623a3: 2024-11-23T14:27:55,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:55,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=169 2024-11-23T14:27:55,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster(4169): Remote procedure done, pid=169 2024-11-23T14:27:55,084 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=169, resume processing ppid=168 2024-11-23T14:27:55,084 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=169, ppid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5710 sec 2024-11-23T14:27:55,086 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees in 1.5750 sec 2024-11-23T14:27:55,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(8855): Flush requested on 59fa5c31bb2297940cc7fa9d0ff623a3 2024-11-23T14:27:55,192 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T14:27:55,192 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 59fa5c31bb2297940cc7fa9d0ff623a3 3/3 column families, dataSize=187.85 KB heapSize=492.94 KB 2024-11-23T14:27:55,196 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:55,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60448 deadline: 1732372135194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:55,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:55,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:55,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:55,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:55,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60476 deadline: 1732372135195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:55,198 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:55,198 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:55,198 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:55,204 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112386dc01fd63a44b289a0baecea87d7f53_59fa5c31bb2297940cc7fa9d0ff623a3 is 50, key is test_row_0/A:col10/1732372074050/Put/seqid=0 2024-11-23T14:27:55,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742495_1671 (size=14994) 2024-11-23T14:27:55,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:55,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:55,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60476 deadline: 1732372135300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:55,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60448 deadline: 1732372135300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:55,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:55,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:55,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:55,302 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:55,302 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:55,302 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:55,510 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:55,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60448 deadline: 1732372135510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:55,511 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:55,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:55,511 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:55,511 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:55,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60476 deadline: 1732372135510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:55,511 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:55,511 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:55,511 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:55,614 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:55,616 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112386dc01fd63a44b289a0baecea87d7f53_59fa5c31bb2297940cc7fa9d0ff623a3 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112386dc01fd63a44b289a0baecea87d7f53_59fa5c31bb2297940cc7fa9d0ff623a3 2024-11-23T14:27:55,617 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/A/f04bdaf421b84b58823b4b737f787d4a, store: [table=TestAcidGuarantees family=A region=59fa5c31bb2297940cc7fa9d0ff623a3] 2024-11-23T14:27:55,617 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/A/f04bdaf421b84b58823b4b737f787d4a is 175, key is test_row_0/A:col10/1732372074050/Put/seqid=0 2024-11-23T14:27:55,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742496_1672 (size=39949) 2024-11-23T14:27:55,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-23T14:27:55,661 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T14:27:55,661 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-23T14:27:55,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] procedure2.ProcedureExecutor(1139): Stored pid=170, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees 2024-11-23T14:27:55,664 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=170, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T14:27:55,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-23T14:27:55,664 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=170, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T14:27:55,664 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=171, ppid=170, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T14:27:55,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-23T14:27:55,816 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-23T14:27:55,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:55,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. as already flushing 2024-11-23T14:27:55,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:55,816 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:55,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:55,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:55,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:55,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60448 deadline: 1732372135820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:55,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:55,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:55,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:55,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60476 deadline: 1732372135820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:55,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:55,821 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:55,822 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:55,822 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:55,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-23T14:27:55,993 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-23T14:27:55,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:55,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. as already flushing 2024-11-23T14:27:55,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:55,993 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:55,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:55,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:56,021 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=367, memsize=64.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/A/f04bdaf421b84b58823b4b737f787d4a 2024-11-23T14:27:56,030 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/B/3dc9ac2e87364f759aa5089902aa739b is 50, key is test_row_0/B:col10/1732372074050/Put/seqid=0 2024-11-23T14:27:56,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742497_1673 (size=12301) 2024-11-23T14:27:56,146 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-23T14:27:56,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:56,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. as already flushing 2024-11-23T14:27:56,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:56,146 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:56,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:56,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:56,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-23T14:27:56,298 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-23T14:27:56,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:56,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. as already flushing 2024-11-23T14:27:56,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:56,299 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:56,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:56,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:56,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:56,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60448 deadline: 1732372136330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:56,331 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:56,331 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:56,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-23T14:27:56,331 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:56,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60476 deadline: 1732372136330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 2024-11-23T14:27:56,331 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 , the old value is region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:56,331 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T14:27:56,331 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., hostname=c6f7b1897123,35385,1732371902767, seqNum=5 because the exception is null or not the one we care about 2024-11-23T14:27:56,451 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-23T14:27:56,451 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=64.85 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/B/3dc9ac2e87364f759aa5089902aa739b 2024-11-23T14:27:56,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:56,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. as already flushing 2024-11-23T14:27:56,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:56,452 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:56,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:56,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T14:27:56,457 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/C/4be7d6a55fe94307b8c61d7b66ae19af is 50, key is test_row_0/C:col10/1732372074050/Put/seqid=0 2024-11-23T14:27:56,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742498_1674 (size=12301) 2024-11-23T14:27:56,470 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=64.85 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/C/4be7d6a55fe94307b8c61d7b66ae19af 2024-11-23T14:27:56,474 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/A/f04bdaf421b84b58823b4b737f787d4a as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/f04bdaf421b84b58823b4b737f787d4a 2024-11-23T14:27:56,477 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/f04bdaf421b84b58823b4b737f787d4a, entries=200, sequenceid=367, filesize=39.0 K 2024-11-23T14:27:56,478 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/B/3dc9ac2e87364f759aa5089902aa739b as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/3dc9ac2e87364f759aa5089902aa739b 2024-11-23T14:27:56,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,481 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/3dc9ac2e87364f759aa5089902aa739b, entries=150, sequenceid=367, filesize=12.0 K 2024-11-23T14:27:56,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,482 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/C/4be7d6a55fe94307b8c61d7b66ae19af as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/4be7d6a55fe94307b8c61d7b66ae19af 2024-11-23T14:27:56,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,485 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/4be7d6a55fe94307b8c61d7b66ae19af, entries=150, sequenceid=367, filesize=12.0 K 2024-11-23T14:27:56,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,485 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~194.56 KB/199230, heapSize ~510.47 KB/522720, currentSize=6.71 KB/6870 for 59fa5c31bb2297940cc7fa9d0ff623a3 in 1293ms, sequenceid=367, compaction requested=true 2024-11-23T14:27:56,485 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 59fa5c31bb2297940cc7fa9d0ff623a3: 2024-11-23T14:27:56,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,486 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:27:56,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,486 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 59fa5c31bb2297940cc7fa9d0ff623a3:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T14:27:56,486 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:27:56,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,486 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:27:56,486 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 59fa5c31bb2297940cc7fa9d0ff623a3:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T14:27:56,486 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:27:56,486 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 59fa5c31bb2297940cc7fa9d0ff623a3:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T14:27:56,486 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:27:56,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,486 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94515 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:27:56,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,486 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1541): 59fa5c31bb2297940cc7fa9d0ff623a3/A is initiating minor compaction (all files) 2024-11-23T14:27:56,486 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 59fa5c31bb2297940cc7fa9d0ff623a3/A in TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:56,487 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/3cf17acce4904588aedc51762e587565, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/207c67522e7044d780f98a713df8af92, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/f04bdaf421b84b58823b4b737f787d4a] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp, totalSize=92.3 K 2024-11-23T14:27:56,487 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:27:56,487 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:56,487 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1541): 59fa5c31bb2297940cc7fa9d0ff623a3/B is initiating minor compaction (all files) 2024-11-23T14:27:56,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,487 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. files: [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/3cf17acce4904588aedc51762e587565, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/207c67522e7044d780f98a713df8af92, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/f04bdaf421b84b58823b4b737f787d4a] 2024-11-23T14:27:56,487 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 59fa5c31bb2297940cc7fa9d0ff623a3/B in TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:56,487 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/b40dbbbdc6f643878145c88bd9a3331e, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/7152ce9a0c314ab58a343ca7ea57ccc3, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/3dc9ac2e87364f759aa5089902aa739b] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp, totalSize=34.4 K 2024-11-23T14:27:56,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,487 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3cf17acce4904588aedc51762e587565, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1732372072497 2024-11-23T14:27:56,487 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting b40dbbbdc6f643878145c88bd9a3331e, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1732372072497 2024-11-23T14:27:56,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,487 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting 207c67522e7044d780f98a713df8af92, keycount=100, bloomtype=ROW, size=22.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1732372072828 2024-11-23T14:27:56,487 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 7152ce9a0c314ab58a343ca7ea57ccc3, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1732372072828 2024-11-23T14:27:56,487 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] compactions.Compactor(225): Compacting f04bdaf421b84b58823b4b737f787d4a, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1732372074047 2024-11-23T14:27:56,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,488 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 3dc9ac2e87364f759aa5089902aa739b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1732372074050 2024-11-23T14:27:56,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,492 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=59fa5c31bb2297940cc7fa9d0ff623a3] 2024-11-23T14:27:56,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,505 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241123c689e47e671148cd8f98c2a7394ecf79_59fa5c31bb2297940cc7fa9d0ff623a3 store=[table=TestAcidGuarantees family=A region=59fa5c31bb2297940cc7fa9d0ff623a3] 2024-11-23T14:27:56,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,506 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 59fa5c31bb2297940cc7fa9d0ff623a3#B#compaction#567 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:27:56,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,506 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/B/cfec14bdb7b74f888714586c3aa943d7 is 50, key is test_row_0/B:col10/1732372074050/Put/seqid=0 2024-11-23T14:27:56,507 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241123c689e47e671148cd8f98c2a7394ecf79_59fa5c31bb2297940cc7fa9d0ff623a3, store=[table=TestAcidGuarantees family=A region=59fa5c31bb2297940cc7fa9d0ff623a3] 2024-11-23T14:27:56,507 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123c689e47e671148cd8f98c2a7394ecf79_59fa5c31bb2297940cc7fa9d0ff623a3 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=59fa5c31bb2297940cc7fa9d0ff623a3] 2024-11-23T14:27:56,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742499_1675 (size=13153) 2024-11-23T14:27:56,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742500_1676 (size=4469) 2024-11-23T14:27:56,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,585 INFO [Thread-2560 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2560. 2024-11-23T14:27:56,585 DEBUG [Thread-2560 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T14:27:56,585 DEBUG [Thread-2560 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:27:56,585 DEBUG [Thread-2560 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:27:56,585 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T14:27:56,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,586 INFO [Thread-2554 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2554. 2024-11-23T14:27:56,586 DEBUG [Thread-2554 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T14:27:56,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,586 DEBUG [Thread-2554 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:27:56,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,586 DEBUG [Thread-2554 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:27:56,586 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T14:27:56,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,594 INFO [Thread-2562 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2562. 2024-11-23T14:27:56,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,594 DEBUG [Thread-2562 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T14:27:56,594 DEBUG [Thread-2562 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:27:56,594 DEBUG [Thread-2562 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:27:56,594 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T14:27:56,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,604 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35385 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-23T14:27:56,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:56,605 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2902): Flushing 59fa5c31bb2297940cc7fa9d0ff623a3 3/3 column families, dataSize=6.71 KB heapSize=18.33 KB 2024-11-23T14:27:56,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411235733d5efdb8a477ea435b13f5c48d63f_59fa5c31bb2297940cc7fa9d0ff623a3 is 50, key is test_row_0/A:col10/1732372075194/Put/seqid=0 2024-11-23T14:27:56,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35385 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:56,617 INFO [Thread-2558 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2558. 2024-11-23T14:27:56,617 DEBUG [Thread-2558 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T14:27:56,617 DEBUG [Thread-2558 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:27:56,617 DEBUG [Thread-2558 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:27:56,617 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T14:27:56,619 INFO [Thread-2556 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2556. 2024-11-23T14:27:56,619 DEBUG [Thread-2556 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T14:27:56,619 DEBUG [Thread-2556 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:27:56,619 DEBUG [Thread-2556 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:27:56,619 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T14:27:56,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742501_1677 (size=7374) 2024-11-23T14:27:56,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-23T14:27:56,913 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 59fa5c31bb2297940cc7fa9d0ff623a3#A#compaction#566 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:27:56,914 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/A/08cd8e6f58d3437ba3b07fb0845a57ef is 175, key is test_row_0/A:col10/1732372074050/Put/seqid=0 2024-11-23T14:27:56,916 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/B/cfec14bdb7b74f888714586c3aa943d7 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/cfec14bdb7b74f888714586c3aa943d7 2024-11-23T14:27:56,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742502_1678 (size=32107) 2024-11-23T14:27:56,919 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 59fa5c31bb2297940cc7fa9d0ff623a3/B of 59fa5c31bb2297940cc7fa9d0ff623a3 into cfec14bdb7b74f888714586c3aa943d7(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:27:56,919 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 59fa5c31bb2297940cc7fa9d0ff623a3: 2024-11-23T14:27:56,919 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., storeName=59fa5c31bb2297940cc7fa9d0ff623a3/B, priority=13, startTime=1732372076486; duration=0sec 2024-11-23T14:27:56,919 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T14:27:56,919 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 59fa5c31bb2297940cc7fa9d0ff623a3:B 2024-11-23T14:27:56,919 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T14:27:56,920 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T14:27:56,920 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1541): 59fa5c31bb2297940cc7fa9d0ff623a3/C is initiating minor compaction (all files) 2024-11-23T14:27:56,920 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 59fa5c31bb2297940cc7fa9d0ff623a3/C in TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:56,920 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/96c5967fe737417ab1a40136c3dac858, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/6468e2c4e7be44059583ea36ee17c7a1, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/4be7d6a55fe94307b8c61d7b66ae19af] into tmpdir=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp, totalSize=34.4 K 2024-11-23T14:27:56,920 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 96c5967fe737417ab1a40136c3dac858, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1732372072497 2024-11-23T14:27:56,921 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 6468e2c4e7be44059583ea36ee17c7a1, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1732372072828 2024-11-23T14:27:56,921 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] compactions.Compactor(225): Compacting 4be7d6a55fe94307b8c61d7b66ae19af, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1732372074050 2024-11-23T14:27:56,925 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 59fa5c31bb2297940cc7fa9d0ff623a3#C#compaction#569 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T14:27:56,926 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/C/ae1ef6dfb50a4f05b2e49c6993c3d3fe is 50, key is test_row_0/C:col10/1732372074050/Put/seqid=0 2024-11-23T14:27:56,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742503_1679 (size=13153) 2024-11-23T14:27:57,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:27:57,028 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411235733d5efdb8a477ea435b13f5c48d63f_59fa5c31bb2297940cc7fa9d0ff623a3 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411235733d5efdb8a477ea435b13f5c48d63f_59fa5c31bb2297940cc7fa9d0ff623a3 2024-11-23T14:27:57,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/A/9587f7aa3dee4045ba25b90f914359c3, store: [table=TestAcidGuarantees family=A region=59fa5c31bb2297940cc7fa9d0ff623a3] 2024-11-23T14:27:57,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/A/9587f7aa3dee4045ba25b90f914359c3 is 175, key is test_row_0/A:col10/1732372075194/Put/seqid=0 2024-11-23T14:27:57,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742504_1680 (size=13865) 2024-11-23T14:27:57,320 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/A/08cd8e6f58d3437ba3b07fb0845a57ef as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/08cd8e6f58d3437ba3b07fb0845a57ef 2024-11-23T14:27:57,323 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 59fa5c31bb2297940cc7fa9d0ff623a3/A of 59fa5c31bb2297940cc7fa9d0ff623a3 into 08cd8e6f58d3437ba3b07fb0845a57ef(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:27:57,323 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 59fa5c31bb2297940cc7fa9d0ff623a3: 2024-11-23T14:27:57,323 INFO [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., storeName=59fa5c31bb2297940cc7fa9d0ff623a3/A, priority=13, startTime=1732372076485; duration=0sec 2024-11-23T14:27:57,323 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:27:57,323 DEBUG [RS:0;c6f7b1897123:35385-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 59fa5c31bb2297940cc7fa9d0ff623a3:A 2024-11-23T14:27:57,332 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/C/ae1ef6dfb50a4f05b2e49c6993c3d3fe as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/ae1ef6dfb50a4f05b2e49c6993c3d3fe 2024-11-23T14:27:57,335 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 59fa5c31bb2297940cc7fa9d0ff623a3/C of 59fa5c31bb2297940cc7fa9d0ff623a3 into ae1ef6dfb50a4f05b2e49c6993c3d3fe(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T14:27:57,335 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 59fa5c31bb2297940cc7fa9d0ff623a3: 2024-11-23T14:27:57,335 INFO [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., storeName=59fa5c31bb2297940cc7fa9d0ff623a3/C, priority=13, startTime=1732372076486; duration=0sec 2024-11-23T14:27:57,336 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T14:27:57,336 DEBUG [RS:0;c6f7b1897123:35385-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 59fa5c31bb2297940cc7fa9d0ff623a3:C 2024-11-23T14:27:57,341 INFO [Thread-2547 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2547. 2024-11-23T14:27:57,341 DEBUG [Thread-2547 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T14:27:57,341 DEBUG [Thread-2547 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:27:57,341 DEBUG [Thread-2547 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:27:57,341 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T14:27:57,351 INFO [Thread-2551 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2551. 2024-11-23T14:27:57,351 DEBUG [Thread-2551 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T14:27:57,351 DEBUG [Thread-2551 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:27:57,351 DEBUG [Thread-2551 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:27:57,351 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T14:27:57,433 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=371, memsize=2.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/A/9587f7aa3dee4045ba25b90f914359c3 2024-11-23T14:27:57,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/B/b2219e42c030491fbf08219521f95a79 is 50, key is test_row_0/B:col10/1732372075194/Put/seqid=0 2024-11-23T14:27:57,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742505_1681 (size=7415) 2024-11-23T14:27:57,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-23T14:27:57,841 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/B/b2219e42c030491fbf08219521f95a79 2024-11-23T14:27:57,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/C/5155c7fd0ec74838a066ade2fa9119cd is 50, key is test_row_0/C:col10/1732372075194/Put/seqid=0 2024-11-23T14:27:57,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742506_1682 (size=7415) 2024-11-23T14:27:58,250 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/C/5155c7fd0ec74838a066ade2fa9119cd 2024-11-23T14:27:58,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/A/9587f7aa3dee4045ba25b90f914359c3 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/9587f7aa3dee4045ba25b90f914359c3 2024-11-23T14:27:58,256 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/9587f7aa3dee4045ba25b90f914359c3, entries=50, sequenceid=371, filesize=13.5 K 2024-11-23T14:27:58,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/B/b2219e42c030491fbf08219521f95a79 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/b2219e42c030491fbf08219521f95a79 2024-11-23T14:27:58,259 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/b2219e42c030491fbf08219521f95a79, entries=50, sequenceid=371, filesize=7.2 K 2024-11-23T14:27:58,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/C/5155c7fd0ec74838a066ade2fa9119cd as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/5155c7fd0ec74838a066ade2fa9119cd 2024-11-23T14:27:58,261 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/5155c7fd0ec74838a066ade2fa9119cd, entries=50, sequenceid=371, filesize=7.2 K 2024-11-23T14:27:58,262 INFO [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(3140): Finished flush of dataSize ~6.71 KB/6870, heapSize ~18.28 KB/18720, currentSize=13.42 KB/13740 for 59fa5c31bb2297940cc7fa9d0ff623a3 in 1657ms, sequenceid=371, compaction requested=false 2024-11-23T14:27:58,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2603): Flush status journal for 59fa5c31bb2297940cc7fa9d0ff623a3: 2024-11-23T14:27:58,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:27:58,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6f7b1897123:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=171 2024-11-23T14:27:58,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster(4169): Remote procedure done, pid=171 2024-11-23T14:27:58,264 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=171, resume processing ppid=170 2024-11-23T14:27:58,264 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=171, ppid=170, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5990 sec 2024-11-23T14:27:58,266 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=170, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees in 2.6030 sec 2024-11-23T14:27:59,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-23T14:27:59,820 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T14:28:00,798 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-23T14:28:04,171 INFO [Thread-2545 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2545. 2024-11-23T14:28:04,171 DEBUG [Thread-2545 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T14:28:04,171 DEBUG [Thread-2545 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:28:04,171 DEBUG [Thread-2545 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:28:04,171 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T14:28:04,182 INFO [Thread-2543 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2543. 2024-11-23T14:28:04,182 DEBUG [Thread-2543 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T14:28:04,182 DEBUG [Thread-2543 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:28:04,182 DEBUG [Thread-2543 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:28:04,182 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T14:28:04,221 INFO [Thread-2549 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2549. 2024-11-23T14:28:04,221 DEBUG [Thread-2549 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T14:28:04,221 DEBUG [Thread-2549 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:28:04,221 DEBUG [Thread-2549 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:28:04,221 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T14:28:04,221 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-23T14:28:04,221 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 33 2024-11-23T14:28:04,221 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 36 2024-11-23T14:28:04,222 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 106 2024-11-23T14:28:04,222 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 25 2024-11-23T14:28:04,222 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 94 2024-11-23T14:28:04,222 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-23T14:28:04,222 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3692 2024-11-23T14:28:04,222 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3626 2024-11-23T14:28:04,222 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3867 2024-11-23T14:28:04,222 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3753 2024-11-23T14:28:04,222 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3620 2024-11-23T14:28:04,222 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-23T14:28:04,222 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-23T14:28:04,222 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T14:28:04,222 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:28:04,222 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:28:04,222 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T14:28:04,223 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-23T14:28:04,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] procedure2.ProcedureExecutor(1139): Stored pid=172, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-23T14:28:04,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-23T14:28:04,225 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732372084225"}]},"ts":"1732372084225"} 2024-11-23T14:28:04,226 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-23T14:28:04,226 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-23T14:28:04,226 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=173, ppid=172, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-23T14:28:04,227 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=174, ppid=173, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=59fa5c31bb2297940cc7fa9d0ff623a3, UNASSIGN}] 2024-11-23T14:28:04,228 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=174, ppid=173, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=59fa5c31bb2297940cc7fa9d0ff623a3, UNASSIGN 2024-11-23T14:28:04,228 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=174 updating hbase:meta row=59fa5c31bb2297940cc7fa9d0ff623a3, regionState=CLOSING, regionLocation=c6f7b1897123,35385,1732371902767 2024-11-23T14:28:04,229 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=174, ppid=173, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=59fa5c31bb2297940cc7fa9d0ff623a3, UNASSIGN because future has completed 2024-11-23T14:28:04,230 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-23T14:28:04,230 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=175, ppid=174, state=RUNNABLE, hasLock=false; CloseRegionProcedure 59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767}] 2024-11-23T14:28:04,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-23T14:28:04,381 INFO [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] handler.UnassignRegionHandler(122): Close 59fa5c31bb2297940cc7fa9d0ff623a3 2024-11-23T14:28:04,381 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-23T14:28:04,382 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1722): Closing 59fa5c31bb2297940cc7fa9d0ff623a3, disabling compactions & flushes 2024-11-23T14:28:04,382 INFO [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:28:04,382 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:28:04,382 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. after waiting 0 ms 2024-11-23T14:28:04,382 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:28:04,382 INFO [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(2902): Flushing 59fa5c31bb2297940cc7fa9d0ff623a3 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-23T14:28:04,387 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411239831449148824f52b71974f510afa297_59fa5c31bb2297940cc7fa9d0ff623a3 is 50, key is test_row_0/A:col10/1732372084180/Put/seqid=0 2024-11-23T14:28:04,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742507_1683 (size=12454) 2024-11-23T14:28:04,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-23T14:28:04,791 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T14:28:04,794 INFO [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411239831449148824f52b71974f510afa297_59fa5c31bb2297940cc7fa9d0ff623a3 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411239831449148824f52b71974f510afa297_59fa5c31bb2297940cc7fa9d0ff623a3 2024-11-23T14:28:04,795 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/A/699cf67687b94babbea01eed9216ab82, store: [table=TestAcidGuarantees family=A region=59fa5c31bb2297940cc7fa9d0ff623a3] 2024-11-23T14:28:04,795 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/A/699cf67687b94babbea01eed9216ab82 is 175, key is test_row_0/A:col10/1732372084180/Put/seqid=0 2024-11-23T14:28:04,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742508_1684 (size=31255) 2024-11-23T14:28:04,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-23T14:28:05,199 INFO [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=382, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/A/699cf67687b94babbea01eed9216ab82 2024-11-23T14:28:05,204 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/B/b6f9688e6ad846d3899ccf8a45ce77f2 is 50, key is test_row_0/B:col10/1732372084180/Put/seqid=0 2024-11-23T14:28:05,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742509_1685 (size=12301) 2024-11-23T14:28:05,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-23T14:28:05,608 INFO [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/B/b6f9688e6ad846d3899ccf8a45ce77f2 2024-11-23T14:28:05,612 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/C/67ca3d29eae34e84b094a45a761bbca6 is 50, key is test_row_0/C:col10/1732372084180/Put/seqid=0 2024-11-23T14:28:05,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742510_1686 (size=12301) 2024-11-23T14:28:06,015 INFO [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/C/67ca3d29eae34e84b094a45a761bbca6 2024-11-23T14:28:06,019 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/A/699cf67687b94babbea01eed9216ab82 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/699cf67687b94babbea01eed9216ab82 2024-11-23T14:28:06,021 INFO [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/699cf67687b94babbea01eed9216ab82, entries=150, sequenceid=382, filesize=30.5 K 2024-11-23T14:28:06,022 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/B/b6f9688e6ad846d3899ccf8a45ce77f2 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/b6f9688e6ad846d3899ccf8a45ce77f2 2024-11-23T14:28:06,024 INFO [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/b6f9688e6ad846d3899ccf8a45ce77f2, entries=150, sequenceid=382, filesize=12.0 K 2024-11-23T14:28:06,024 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/.tmp/C/67ca3d29eae34e84b094a45a761bbca6 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/67ca3d29eae34e84b094a45a761bbca6 2024-11-23T14:28:06,026 INFO [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/67ca3d29eae34e84b094a45a761bbca6, entries=150, sequenceid=382, filesize=12.0 K 2024-11-23T14:28:06,027 INFO [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 59fa5c31bb2297940cc7fa9d0ff623a3 in 1645ms, sequenceid=382, compaction requested=true 2024-11-23T14:28:06,027 DEBUG [StoreCloser-TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/f96f960d09894b3fb8c4032161828808, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/8e5c4798f8fd463495bc04d01ac121f9, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/b4c583d41d4b47938ca835dd1bd635d6, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/b133da13ac6b49538a06d643f6e43d41, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/b28956adc03f49fe85d832ee99a4ff80, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/1b9b7b6da2aa41f28e6a6141bc5edab8, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/3213e5165612488eb54c9f871f563ef6, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/74aa33b45610494dbcb5e7a5c2731685, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/449f26e910c649ecb57b375b225d4601, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/4dc72b98072644688f40348257298251, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/1e899a3f480b4a9b83d0d6e4dfa3a3a4, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/2120a4a3dfc74f5598eb0f6d76abb3b5, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/66ff6aeeeb674cb29adac00f298dd2c9, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/ce5a8064bdea4ee8b3158a27940a8c46, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/0f325658aba0481286dd38abe30194cc, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/447c0d6ae52e48189513d8d8b65d7fdd, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/8804ebd89cd74c8da5ca2a31b2381031, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/378dd764800d4a679a410bab935c87ae, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/bde9752367774277a70ef6d28dbe7666, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/10533be2358a4af9b99b09aecbcdde04, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/6902365158414bb29509a2f2a1e4e5b9, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/03593d1c5a3c42e6aa86693dde102fe0, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/3cf17acce4904588aedc51762e587565, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/207c67522e7044d780f98a713df8af92, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/f04bdaf421b84b58823b4b737f787d4a] to archive 2024-11-23T14:28:06,028 DEBUG [StoreCloser-TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-23T14:28:06,030 DEBUG [StoreCloser-TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/f96f960d09894b3fb8c4032161828808 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/f96f960d09894b3fb8c4032161828808 2024-11-23T14:28:06,031 DEBUG [StoreCloser-TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/8e5c4798f8fd463495bc04d01ac121f9 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/8e5c4798f8fd463495bc04d01ac121f9 2024-11-23T14:28:06,032 DEBUG [StoreCloser-TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/b4c583d41d4b47938ca835dd1bd635d6 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/b4c583d41d4b47938ca835dd1bd635d6 2024-11-23T14:28:06,033 DEBUG [StoreCloser-TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/b133da13ac6b49538a06d643f6e43d41 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/b133da13ac6b49538a06d643f6e43d41 2024-11-23T14:28:06,034 DEBUG [StoreCloser-TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/b28956adc03f49fe85d832ee99a4ff80 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/b28956adc03f49fe85d832ee99a4ff80 2024-11-23T14:28:06,034 DEBUG [StoreCloser-TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/1b9b7b6da2aa41f28e6a6141bc5edab8 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/1b9b7b6da2aa41f28e6a6141bc5edab8 2024-11-23T14:28:06,035 DEBUG [StoreCloser-TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/3213e5165612488eb54c9f871f563ef6 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/3213e5165612488eb54c9f871f563ef6 2024-11-23T14:28:06,036 DEBUG [StoreCloser-TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/74aa33b45610494dbcb5e7a5c2731685 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/74aa33b45610494dbcb5e7a5c2731685 2024-11-23T14:28:06,037 DEBUG [StoreCloser-TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/449f26e910c649ecb57b375b225d4601 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/449f26e910c649ecb57b375b225d4601 2024-11-23T14:28:06,037 DEBUG [StoreCloser-TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/4dc72b98072644688f40348257298251 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/4dc72b98072644688f40348257298251 2024-11-23T14:28:06,038 DEBUG [StoreCloser-TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/1e899a3f480b4a9b83d0d6e4dfa3a3a4 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/1e899a3f480b4a9b83d0d6e4dfa3a3a4 2024-11-23T14:28:06,038 DEBUG [StoreCloser-TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/2120a4a3dfc74f5598eb0f6d76abb3b5 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/2120a4a3dfc74f5598eb0f6d76abb3b5 2024-11-23T14:28:06,039 DEBUG [StoreCloser-TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/66ff6aeeeb674cb29adac00f298dd2c9 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/66ff6aeeeb674cb29adac00f298dd2c9 2024-11-23T14:28:06,040 DEBUG [StoreCloser-TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/ce5a8064bdea4ee8b3158a27940a8c46 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/ce5a8064bdea4ee8b3158a27940a8c46 2024-11-23T14:28:06,041 DEBUG [StoreCloser-TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/0f325658aba0481286dd38abe30194cc to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/0f325658aba0481286dd38abe30194cc 2024-11-23T14:28:06,041 DEBUG [StoreCloser-TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/447c0d6ae52e48189513d8d8b65d7fdd to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/447c0d6ae52e48189513d8d8b65d7fdd 2024-11-23T14:28:06,042 DEBUG [StoreCloser-TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/8804ebd89cd74c8da5ca2a31b2381031 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/8804ebd89cd74c8da5ca2a31b2381031 2024-11-23T14:28:06,043 DEBUG [StoreCloser-TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/378dd764800d4a679a410bab935c87ae to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/378dd764800d4a679a410bab935c87ae 2024-11-23T14:28:06,044 DEBUG [StoreCloser-TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/bde9752367774277a70ef6d28dbe7666 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/bde9752367774277a70ef6d28dbe7666 2024-11-23T14:28:06,044 DEBUG [StoreCloser-TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/10533be2358a4af9b99b09aecbcdde04 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/10533be2358a4af9b99b09aecbcdde04 2024-11-23T14:28:06,045 DEBUG [StoreCloser-TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/6902365158414bb29509a2f2a1e4e5b9 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/6902365158414bb29509a2f2a1e4e5b9 2024-11-23T14:28:06,046 DEBUG [StoreCloser-TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/03593d1c5a3c42e6aa86693dde102fe0 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/03593d1c5a3c42e6aa86693dde102fe0 2024-11-23T14:28:06,046 DEBUG [StoreCloser-TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/3cf17acce4904588aedc51762e587565 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/3cf17acce4904588aedc51762e587565 2024-11-23T14:28:06,047 DEBUG [StoreCloser-TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/207c67522e7044d780f98a713df8af92 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/207c67522e7044d780f98a713df8af92 2024-11-23T14:28:06,047 DEBUG [StoreCloser-TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/f04bdaf421b84b58823b4b737f787d4a to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/f04bdaf421b84b58823b4b737f787d4a 2024-11-23T14:28:06,049 DEBUG [StoreCloser-TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/0d45164106f64a3991d287fe0c8058e9, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/08b1fa65873c41528b538b75599d649f, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/ebb73acb95214f769491fbfccb36903f, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/3056c23946c14b5794da0a1a93560a1a, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/dcfa774201a24337986d60ee24efdde8, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/8954bad5b3074923af22fffb8d9867f1, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/f007742cef554fd4b1690dbf00ff2c47, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/2cbc817bbfdf4d1296937bd0c6b32c37, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/ab3a75da0f4e498ebc1de46459d34cf3, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/2b3ae18e14a34736bb118604de5793c3, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/bb5eefb597174834a05e4b085ac0d7bc, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/77e7f8fd88a046828f214d716fcb3fb3, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/0c6163bf21d04f04baf227da10ebadd3, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/b55417cdb155411c9e8942d41c50565d, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/49f255242a6244e6a3ae655bfe087ed1, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/e69c6aff3f1a4c8aa94ff1b38f2a7038, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/9de15cb0e78b4b979ed6e36b3a919b11, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/b762fa7654ad4569bd82066db2d658b1, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/6171f780d0ed44b3b76aad1993631682, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/9c53898f74fe45d1aecbf01cf642e6d0, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/c42f4b5501414add990c4c7c67f8cffd, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/b40dbbbdc6f643878145c88bd9a3331e, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/6239373ad90c451eb34c7fd51fb48300, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/7152ce9a0c314ab58a343ca7ea57ccc3, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/3dc9ac2e87364f759aa5089902aa739b] to archive 2024-11-23T14:28:06,049 DEBUG [StoreCloser-TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-23T14:28:06,050 DEBUG [StoreCloser-TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/0d45164106f64a3991d287fe0c8058e9 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/0d45164106f64a3991d287fe0c8058e9 2024-11-23T14:28:06,051 DEBUG [StoreCloser-TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/08b1fa65873c41528b538b75599d649f to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/08b1fa65873c41528b538b75599d649f 2024-11-23T14:28:06,051 DEBUG [StoreCloser-TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/ebb73acb95214f769491fbfccb36903f to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/ebb73acb95214f769491fbfccb36903f 2024-11-23T14:28:06,052 DEBUG [StoreCloser-TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/3056c23946c14b5794da0a1a93560a1a to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/3056c23946c14b5794da0a1a93560a1a 2024-11-23T14:28:06,053 DEBUG [StoreCloser-TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/dcfa774201a24337986d60ee24efdde8 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/dcfa774201a24337986d60ee24efdde8 2024-11-23T14:28:06,053 DEBUG [StoreCloser-TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/8954bad5b3074923af22fffb8d9867f1 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/8954bad5b3074923af22fffb8d9867f1 2024-11-23T14:28:06,054 DEBUG [StoreCloser-TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/f007742cef554fd4b1690dbf00ff2c47 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/f007742cef554fd4b1690dbf00ff2c47 2024-11-23T14:28:06,054 DEBUG [StoreCloser-TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/2cbc817bbfdf4d1296937bd0c6b32c37 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/2cbc817bbfdf4d1296937bd0c6b32c37 2024-11-23T14:28:06,055 DEBUG [StoreCloser-TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/ab3a75da0f4e498ebc1de46459d34cf3 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/ab3a75da0f4e498ebc1de46459d34cf3 2024-11-23T14:28:06,056 DEBUG [StoreCloser-TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/2b3ae18e14a34736bb118604de5793c3 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/2b3ae18e14a34736bb118604de5793c3 2024-11-23T14:28:06,056 DEBUG [StoreCloser-TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/bb5eefb597174834a05e4b085ac0d7bc to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/bb5eefb597174834a05e4b085ac0d7bc 2024-11-23T14:28:06,057 DEBUG [StoreCloser-TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/77e7f8fd88a046828f214d716fcb3fb3 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/77e7f8fd88a046828f214d716fcb3fb3 2024-11-23T14:28:06,057 DEBUG [StoreCloser-TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/0c6163bf21d04f04baf227da10ebadd3 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/0c6163bf21d04f04baf227da10ebadd3 2024-11-23T14:28:06,058 DEBUG [StoreCloser-TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/b55417cdb155411c9e8942d41c50565d to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/b55417cdb155411c9e8942d41c50565d 2024-11-23T14:28:06,058 DEBUG [StoreCloser-TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/49f255242a6244e6a3ae655bfe087ed1 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/49f255242a6244e6a3ae655bfe087ed1 2024-11-23T14:28:06,059 DEBUG [StoreCloser-TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/e69c6aff3f1a4c8aa94ff1b38f2a7038 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/e69c6aff3f1a4c8aa94ff1b38f2a7038 2024-11-23T14:28:06,060 DEBUG [StoreCloser-TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/9de15cb0e78b4b979ed6e36b3a919b11 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/9de15cb0e78b4b979ed6e36b3a919b11 2024-11-23T14:28:06,060 DEBUG [StoreCloser-TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/b762fa7654ad4569bd82066db2d658b1 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/b762fa7654ad4569bd82066db2d658b1 2024-11-23T14:28:06,061 DEBUG [StoreCloser-TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/6171f780d0ed44b3b76aad1993631682 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/6171f780d0ed44b3b76aad1993631682 2024-11-23T14:28:06,061 DEBUG [StoreCloser-TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/9c53898f74fe45d1aecbf01cf642e6d0 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/9c53898f74fe45d1aecbf01cf642e6d0 2024-11-23T14:28:06,062 DEBUG [StoreCloser-TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/c42f4b5501414add990c4c7c67f8cffd to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/c42f4b5501414add990c4c7c67f8cffd 2024-11-23T14:28:06,063 DEBUG [StoreCloser-TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/b40dbbbdc6f643878145c88bd9a3331e to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/b40dbbbdc6f643878145c88bd9a3331e 2024-11-23T14:28:06,063 DEBUG [StoreCloser-TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/6239373ad90c451eb34c7fd51fb48300 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/6239373ad90c451eb34c7fd51fb48300 2024-11-23T14:28:06,064 DEBUG [StoreCloser-TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/7152ce9a0c314ab58a343ca7ea57ccc3 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/7152ce9a0c314ab58a343ca7ea57ccc3 2024-11-23T14:28:06,065 DEBUG [StoreCloser-TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/3dc9ac2e87364f759aa5089902aa739b to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/3dc9ac2e87364f759aa5089902aa739b 2024-11-23T14:28:06,066 DEBUG [StoreCloser-TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/8260e3a92bb0437e8b63f09846d93390, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/ca9cb7616cd54e62af8f8c1c7ff85b7b, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/4ebe0ca8dc9b494c9730827905c76429, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/fa89e087536c40069d544160d5d019f0, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/8664b2065ba14f07b640dd8aa9bd854a, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/4dc80c9b40594fe7961b4ceb21c7085e, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/e1e7ef02a7e24c5d85569097ba02520f, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/92bb65620a8544bfb033933e9ffe7a49, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/bc94a67f48194873b251663a454bbb98, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/ddd5a451897048c19e89a9cf95b2490b, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/741430a73c964c31a1511d257d5624d5, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/a3e554e5af944f579222ce5500d63cc0, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/c4d727d1f5124394829ddaccf4b8ee67, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/7c6a4e542bbf4e7d84e13fec89c6e1ec, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/9f3e78c2bde44efc95dd7962ab74b55b, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/158e090468ce41f0aa1ae34b30c85fb5, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/ec2b9d7e50ee4d85a54717cd5c8e7dc9, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/00208754d7194f979d90eedecad512a4, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/cc34ce52d7934a4fa7bcb59213027677, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/5bb6e0acbb1a443c9936990a30e8490e, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/1c9896c1f94a4fb6aadaac2fc0fe2344, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/96c5967fe737417ab1a40136c3dac858, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/a0047a773c7d4edbaf2c09476628e467, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/6468e2c4e7be44059583ea36ee17c7a1, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/4be7d6a55fe94307b8c61d7b66ae19af] to archive 2024-11-23T14:28:06,066 DEBUG [StoreCloser-TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-23T14:28:06,067 DEBUG [StoreCloser-TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/8260e3a92bb0437e8b63f09846d93390 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/8260e3a92bb0437e8b63f09846d93390 2024-11-23T14:28:06,068 DEBUG [StoreCloser-TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/ca9cb7616cd54e62af8f8c1c7ff85b7b to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/ca9cb7616cd54e62af8f8c1c7ff85b7b 2024-11-23T14:28:06,068 DEBUG [StoreCloser-TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/4ebe0ca8dc9b494c9730827905c76429 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/4ebe0ca8dc9b494c9730827905c76429 2024-11-23T14:28:06,069 DEBUG [StoreCloser-TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/fa89e087536c40069d544160d5d019f0 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/fa89e087536c40069d544160d5d019f0 2024-11-23T14:28:06,070 DEBUG [StoreCloser-TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/8664b2065ba14f07b640dd8aa9bd854a to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/8664b2065ba14f07b640dd8aa9bd854a 2024-11-23T14:28:06,070 DEBUG [StoreCloser-TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/4dc80c9b40594fe7961b4ceb21c7085e to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/4dc80c9b40594fe7961b4ceb21c7085e 2024-11-23T14:28:06,071 DEBUG [StoreCloser-TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/e1e7ef02a7e24c5d85569097ba02520f to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/e1e7ef02a7e24c5d85569097ba02520f 2024-11-23T14:28:06,071 DEBUG [StoreCloser-TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/92bb65620a8544bfb033933e9ffe7a49 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/92bb65620a8544bfb033933e9ffe7a49 2024-11-23T14:28:06,072 DEBUG [StoreCloser-TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/bc94a67f48194873b251663a454bbb98 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/bc94a67f48194873b251663a454bbb98 2024-11-23T14:28:06,073 DEBUG [StoreCloser-TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/ddd5a451897048c19e89a9cf95b2490b to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/ddd5a451897048c19e89a9cf95b2490b 2024-11-23T14:28:06,073 DEBUG [StoreCloser-TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/741430a73c964c31a1511d257d5624d5 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/741430a73c964c31a1511d257d5624d5 2024-11-23T14:28:06,074 DEBUG [StoreCloser-TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/a3e554e5af944f579222ce5500d63cc0 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/a3e554e5af944f579222ce5500d63cc0 2024-11-23T14:28:06,074 DEBUG [StoreCloser-TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/c4d727d1f5124394829ddaccf4b8ee67 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/c4d727d1f5124394829ddaccf4b8ee67 2024-11-23T14:28:06,075 DEBUG [StoreCloser-TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/7c6a4e542bbf4e7d84e13fec89c6e1ec to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/7c6a4e542bbf4e7d84e13fec89c6e1ec 2024-11-23T14:28:06,076 DEBUG [StoreCloser-TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/9f3e78c2bde44efc95dd7962ab74b55b to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/9f3e78c2bde44efc95dd7962ab74b55b 2024-11-23T14:28:06,076 DEBUG [StoreCloser-TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/158e090468ce41f0aa1ae34b30c85fb5 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/158e090468ce41f0aa1ae34b30c85fb5 2024-11-23T14:28:06,077 DEBUG [StoreCloser-TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/ec2b9d7e50ee4d85a54717cd5c8e7dc9 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/ec2b9d7e50ee4d85a54717cd5c8e7dc9 2024-11-23T14:28:06,078 DEBUG [StoreCloser-TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/00208754d7194f979d90eedecad512a4 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/00208754d7194f979d90eedecad512a4 2024-11-23T14:28:06,079 DEBUG [StoreCloser-TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/cc34ce52d7934a4fa7bcb59213027677 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/cc34ce52d7934a4fa7bcb59213027677 2024-11-23T14:28:06,079 DEBUG [StoreCloser-TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/5bb6e0acbb1a443c9936990a30e8490e to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/5bb6e0acbb1a443c9936990a30e8490e 2024-11-23T14:28:06,080 DEBUG [StoreCloser-TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/1c9896c1f94a4fb6aadaac2fc0fe2344 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/1c9896c1f94a4fb6aadaac2fc0fe2344 2024-11-23T14:28:06,081 DEBUG [StoreCloser-TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/96c5967fe737417ab1a40136c3dac858 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/96c5967fe737417ab1a40136c3dac858 2024-11-23T14:28:06,081 DEBUG [StoreCloser-TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/a0047a773c7d4edbaf2c09476628e467 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/a0047a773c7d4edbaf2c09476628e467 2024-11-23T14:28:06,082 DEBUG [StoreCloser-TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/6468e2c4e7be44059583ea36ee17c7a1 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/6468e2c4e7be44059583ea36ee17c7a1 2024-11-23T14:28:06,083 DEBUG [StoreCloser-TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/4be7d6a55fe94307b8c61d7b66ae19af to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/4be7d6a55fe94307b8c61d7b66ae19af 2024-11-23T14:28:06,089 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/recovered.edits/385.seqid, newMaxSeqId=385, maxSeqId=4 2024-11-23T14:28:06,090 INFO [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. 2024-11-23T14:28:06,090 DEBUG [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1676): Region close journal for 59fa5c31bb2297940cc7fa9d0ff623a3: Waiting for close lock at 1732372084382Running coprocessor pre-close hooks at 1732372084382Disabling compacts and flushes for region at 1732372084382Disabling writes for close at 1732372084382Obtaining lock to block concurrent updates at 1732372084382Preparing flush snapshotting stores in 59fa5c31bb2297940cc7fa9d0ff623a3 at 1732372084382Finished memstore snapshotting TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1732372084382Flushing stores of TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3. at 1732372084382Flushing 59fa5c31bb2297940cc7fa9d0ff623a3/A: creating writer at 1732372084382Flushing mob file 59fa5c31bb2297940cc7fa9d0ff623a3/A: appending metadata at 1732372084387 (+5 ms)Flushing mob file 59fa5c31bb2297940cc7fa9d0ff623a3/A: closing flushed file at 1732372084387Flushing 59fa5c31bb2297940cc7fa9d0ff623a3/A: appending metadata at 1732372084795 (+408 ms)Flushing 59fa5c31bb2297940cc7fa9d0ff623a3/A: closing flushed file at 1732372084795Flushing 59fa5c31bb2297940cc7fa9d0ff623a3/B: creating writer at 1732372085202 (+407 ms)Flushing 59fa5c31bb2297940cc7fa9d0ff623a3/B: appending metadata at 1732372085204 (+2 ms)Flushing 59fa5c31bb2297940cc7fa9d0ff623a3/B: closing flushed file at 1732372085204Flushing 59fa5c31bb2297940cc7fa9d0ff623a3/C: creating writer at 1732372085610 (+406 ms)Flushing 59fa5c31bb2297940cc7fa9d0ff623a3/C: appending metadata at 1732372085612 (+2 ms)Flushing 59fa5c31bb2297940cc7fa9d0ff623a3/C: closing flushed file at 1732372085612Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5a0eeeee: reopening flushed file at 1732372086018 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2b342fab: reopening flushed file at 1732372086021 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@8213fa9: reopening flushed file at 1732372086024 (+3 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 59fa5c31bb2297940cc7fa9d0ff623a3 in 1645ms, sequenceid=382, compaction requested=true at 1732372086027 (+3 ms)Writing region close event to WAL at 1732372086084 (+57 ms)Running coprocessor post-close hooks at 1732372086090 (+6 ms)Closed at 1732372086090 2024-11-23T14:28:06,091 INFO [RS_CLOSE_REGION-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] handler.UnassignRegionHandler(157): Closed 59fa5c31bb2297940cc7fa9d0ff623a3 2024-11-23T14:28:06,092 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=174 updating hbase:meta row=59fa5c31bb2297940cc7fa9d0ff623a3, regionState=CLOSED 2024-11-23T14:28:06,093 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=175, ppid=174, state=RUNNABLE, hasLock=false; CloseRegionProcedure 59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 because future has completed 2024-11-23T14:28:06,095 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=175, resume processing ppid=174 2024-11-23T14:28:06,095 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=175, ppid=174, state=SUCCESS, hasLock=false; CloseRegionProcedure 59fa5c31bb2297940cc7fa9d0ff623a3, server=c6f7b1897123,35385,1732371902767 in 1.8640 sec 2024-11-23T14:28:06,096 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=174, resume processing ppid=173 2024-11-23T14:28:06,096 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=174, ppid=173, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=59fa5c31bb2297940cc7fa9d0ff623a3, UNASSIGN in 1.8680 sec 2024-11-23T14:28:06,098 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=173, resume processing ppid=172 2024-11-23T14:28:06,098 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=173, ppid=172, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8710 sec 2024-11-23T14:28:06,099 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732372086099"}]},"ts":"1732372086099"} 2024-11-23T14:28:06,100 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-23T14:28:06,100 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-23T14:28:06,102 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=172, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8780 sec 2024-11-23T14:28:06,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-23T14:28:06,370 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-23T14:28:06,370 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-23T14:28:06,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] procedure2.ProcedureExecutor(1139): Stored pid=176, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T14:28:06,372 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=176, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T14:28:06,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-23T14:28:06,373 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=176, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T14:28:06,375 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3 2024-11-23T14:28:06,377 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A, FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B, FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C, FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/recovered.edits] 2024-11-23T14:28:06,379 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/08cd8e6f58d3437ba3b07fb0845a57ef to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/08cd8e6f58d3437ba3b07fb0845a57ef 2024-11-23T14:28:06,380 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/699cf67687b94babbea01eed9216ab82 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/699cf67687b94babbea01eed9216ab82 2024-11-23T14:28:06,381 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/9587f7aa3dee4045ba25b90f914359c3 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/A/9587f7aa3dee4045ba25b90f914359c3 2024-11-23T14:28:06,382 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/b2219e42c030491fbf08219521f95a79 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/b2219e42c030491fbf08219521f95a79 2024-11-23T14:28:06,383 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/b6f9688e6ad846d3899ccf8a45ce77f2 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/b6f9688e6ad846d3899ccf8a45ce77f2 2024-11-23T14:28:06,384 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/cfec14bdb7b74f888714586c3aa943d7 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/B/cfec14bdb7b74f888714586c3aa943d7 2024-11-23T14:28:06,385 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/5155c7fd0ec74838a066ade2fa9119cd to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/5155c7fd0ec74838a066ade2fa9119cd 2024-11-23T14:28:06,386 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/67ca3d29eae34e84b094a45a761bbca6 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/67ca3d29eae34e84b094a45a761bbca6 2024-11-23T14:28:06,387 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/ae1ef6dfb50a4f05b2e49c6993c3d3fe to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/C/ae1ef6dfb50a4f05b2e49c6993c3d3fe 2024-11-23T14:28:06,388 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/recovered.edits/385.seqid to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3/recovered.edits/385.seqid 2024-11-23T14:28:06,388 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/default/TestAcidGuarantees/59fa5c31bb2297940cc7fa9d0ff623a3 2024-11-23T14:28:06,389 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-23T14:28:06,389 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-23T14:28:06,389 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-23T14:28:06,391 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123003a5e59486143189aec5ca878d837c0_59fa5c31bb2297940cc7fa9d0ff623a3 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123003a5e59486143189aec5ca878d837c0_59fa5c31bb2297940cc7fa9d0ff623a3 2024-11-23T14:28:06,392 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411230078cb74fdb047c2aceae71a871a5081_59fa5c31bb2297940cc7fa9d0ff623a3 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411230078cb74fdb047c2aceae71a871a5081_59fa5c31bb2297940cc7fa9d0ff623a3 2024-11-23T14:28:06,393 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411231573ad2935064781b794fbd2d3673792_59fa5c31bb2297940cc7fa9d0ff623a3 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411231573ad2935064781b794fbd2d3673792_59fa5c31bb2297940cc7fa9d0ff623a3 2024-11-23T14:28:06,394 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411231b1b44e675f54fcf86f239711b07e282_59fa5c31bb2297940cc7fa9d0ff623a3 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411231b1b44e675f54fcf86f239711b07e282_59fa5c31bb2297940cc7fa9d0ff623a3 2024-11-23T14:28:06,395 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123261bdf9809c04d008a99f50a112f3173_59fa5c31bb2297940cc7fa9d0ff623a3 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123261bdf9809c04d008a99f50a112f3173_59fa5c31bb2297940cc7fa9d0ff623a3 2024-11-23T14:28:06,395 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411235733d5efdb8a477ea435b13f5c48d63f_59fa5c31bb2297940cc7fa9d0ff623a3 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411235733d5efdb8a477ea435b13f5c48d63f_59fa5c31bb2297940cc7fa9d0ff623a3 2024-11-23T14:28:06,396 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411235abff9c1f6e149f2813b81115416a8b4_59fa5c31bb2297940cc7fa9d0ff623a3 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411235abff9c1f6e149f2813b81115416a8b4_59fa5c31bb2297940cc7fa9d0ff623a3 2024-11-23T14:28:06,397 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411236fd0fad327cd4de2a37a5ceef7e706d3_59fa5c31bb2297940cc7fa9d0ff623a3 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411236fd0fad327cd4de2a37a5ceef7e706d3_59fa5c31bb2297940cc7fa9d0ff623a3 2024-11-23T14:28:06,398 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123860151f05ea849dda17876530533af8e_59fa5c31bb2297940cc7fa9d0ff623a3 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123860151f05ea849dda17876530533af8e_59fa5c31bb2297940cc7fa9d0ff623a3 2024-11-23T14:28:06,399 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112386dc01fd63a44b289a0baecea87d7f53_59fa5c31bb2297940cc7fa9d0ff623a3 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112386dc01fd63a44b289a0baecea87d7f53_59fa5c31bb2297940cc7fa9d0ff623a3 2024-11-23T14:28:06,399 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411239249591aaa384ab9881f436bead6f5a5_59fa5c31bb2297940cc7fa9d0ff623a3 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411239249591aaa384ab9881f436bead6f5a5_59fa5c31bb2297940cc7fa9d0ff623a3 2024-11-23T14:28:06,400 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411239831449148824f52b71974f510afa297_59fa5c31bb2297940cc7fa9d0ff623a3 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411239831449148824f52b71974f510afa297_59fa5c31bb2297940cc7fa9d0ff623a3 2024-11-23T14:28:06,401 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411239df7de9cee504fb8ba94ca51f3889028_59fa5c31bb2297940cc7fa9d0ff623a3 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411239df7de9cee504fb8ba94ca51f3889028_59fa5c31bb2297940cc7fa9d0ff623a3 2024-11-23T14:28:06,402 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123b7d6a2d7172347f5a7d618c8ddae90dc_59fa5c31bb2297940cc7fa9d0ff623a3 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123b7d6a2d7172347f5a7d618c8ddae90dc_59fa5c31bb2297940cc7fa9d0ff623a3 2024-11-23T14:28:06,403 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123c28c2ff191f14201baf85924f5caf1c2_59fa5c31bb2297940cc7fa9d0ff623a3 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123c28c2ff191f14201baf85924f5caf1c2_59fa5c31bb2297940cc7fa9d0ff623a3 2024-11-23T14:28:06,403 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123d840820bf46641958a2dce555164532b_59fa5c31bb2297940cc7fa9d0ff623a3 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123d840820bf46641958a2dce555164532b_59fa5c31bb2297940cc7fa9d0ff623a3 2024-11-23T14:28:06,404 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123e6afb821ebe64211abbde642cd16eb77_59fa5c31bb2297940cc7fa9d0ff623a3 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123e6afb821ebe64211abbde642cd16eb77_59fa5c31bb2297940cc7fa9d0ff623a3 2024-11-23T14:28:06,405 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123ebedf3bbbd654d9b9a91ed6b5470cc49_59fa5c31bb2297940cc7fa9d0ff623a3 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123ebedf3bbbd654d9b9a91ed6b5470cc49_59fa5c31bb2297940cc7fa9d0ff623a3 2024-11-23T14:28:06,406 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123f6fa48ea4fc043c39f68ef50dce7372e_59fa5c31bb2297940cc7fa9d0ff623a3 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123f6fa48ea4fc043c39f68ef50dce7372e_59fa5c31bb2297940cc7fa9d0ff623a3 2024-11-23T14:28:06,407 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123fcde1044f5c74f4d9b162a91ba40d25c_59fa5c31bb2297940cc7fa9d0ff623a3 to hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123fcde1044f5c74f4d9b162a91ba40d25c_59fa5c31bb2297940cc7fa9d0ff623a3 2024-11-23T14:28:06,407 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-23T14:28:06,409 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=176, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T14:28:06,411 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-23T14:28:06,413 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-23T14:28:06,414 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=176, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T14:28:06,414 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-23T14:28:06,414 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732372086414"}]},"ts":"9223372036854775807"} 2024-11-23T14:28:06,416 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-23T14:28:06,416 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 59fa5c31bb2297940cc7fa9d0ff623a3, NAME => 'TestAcidGuarantees,,1732372053798.59fa5c31bb2297940cc7fa9d0ff623a3.', STARTKEY => '', ENDKEY => ''}] 2024-11-23T14:28:06,416 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-23T14:28:06,416 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732372086416"}]},"ts":"9223372036854775807"} 2024-11-23T14:28:06,417 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-23T14:28:06,418 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=176, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T14:28:06,419 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=176, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 47 msec 2024-11-23T14:28:06,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-23T14:28:06,480 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-23T14:28:06,480 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-23T14:28:06,488 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=237 (was 240), OpenFileDescriptor=453 (was 447) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=507 (was 552), ProcessCount=11 (was 11), AvailableMemoryMB=3013 (was 3102) 2024-11-23T14:28:06,488 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-23T14:28:06,488 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-23T14:28:06,488 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T14:28:06,488 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:28:06,488 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:28:06,488 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T14:28:06,488 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-23T14:28:06,488 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=2117780854, stopped=false 2024-11-23T14:28:06,489 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=c6f7b1897123,38947,1732371901705 2024-11-23T14:28:06,555 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38947-0x101681565570000, quorum=127.0.0.1:50153, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-23T14:28:06,555 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35385-0x101681565570001, quorum=127.0.0.1:50153, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-23T14:28:06,555 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35385-0x101681565570001, quorum=127.0.0.1:50153, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T14:28:06,555 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38947-0x101681565570000, quorum=127.0.0.1:50153, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T14:28:06,555 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-23T14:28:06,555 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-23T14:28:06,555 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T14:28:06,555 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:28:06,556 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'c6f7b1897123,35385,1732371902767' ***** 2024-11-23T14:28:06,556 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-23T14:28:06,556 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:35385-0x101681565570001, quorum=127.0.0.1:50153, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T14:28:06,556 INFO [RS:0;c6f7b1897123:35385 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-23T14:28:06,556 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-23T14:28:06,556 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:38947-0x101681565570000, quorum=127.0.0.1:50153, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T14:28:06,557 INFO [RS:0;c6f7b1897123:35385 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-23T14:28:06,557 INFO [RS:0;c6f7b1897123:35385 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-23T14:28:06,557 INFO [RS:0;c6f7b1897123:35385 {}] regionserver.HRegionServer(959): stopping server c6f7b1897123,35385,1732371902767 2024-11-23T14:28:06,557 INFO [RS:0;c6f7b1897123:35385 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-23T14:28:06,557 INFO [RS:0;c6f7b1897123:35385 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;c6f7b1897123:35385. 2024-11-23T14:28:06,557 DEBUG [RS:0;c6f7b1897123:35385 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T14:28:06,557 DEBUG [RS:0;c6f7b1897123:35385 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:28:06,557 INFO [RS:0;c6f7b1897123:35385 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-23T14:28:06,557 INFO [RS:0;c6f7b1897123:35385 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-23T14:28:06,558 INFO [RS:0;c6f7b1897123:35385 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-23T14:28:06,558 INFO [RS:0;c6f7b1897123:35385 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-23T14:28:06,558 INFO [RS:0;c6f7b1897123:35385 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-23T14:28:06,558 DEBUG [RS:0;c6f7b1897123:35385 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-23T14:28:06,558 DEBUG [RS_CLOSE_META-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-23T14:28:06,558 INFO [RS_CLOSE_META-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-23T14:28:06,559 DEBUG [RS_CLOSE_META-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-23T14:28:06,559 DEBUG [RS_CLOSE_META-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-23T14:28:06,559 DEBUG [RS_CLOSE_META-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-23T14:28:06,559 DEBUG [RS:0;c6f7b1897123:35385 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-23T14:28:06,559 INFO [RS_CLOSE_META-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-23T14:28:06,579 DEBUG [RS_CLOSE_META-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/hbase/meta/1588230740/.tmp/info/5b5a8d2725b34605b2ff463dda1e6f42 is 95, key is TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe./info:/1732371934937/DeleteFamily/seqid=0 2024-11-23T14:28:06,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742511_1687 (size=6514) 2024-11-23T14:28:06,646 INFO [regionserver/c6f7b1897123:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-23T14:28:06,759 DEBUG [RS:0;c6f7b1897123:35385 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-23T14:28:06,811 INFO [regionserver/c6f7b1897123:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-23T14:28:06,811 INFO [regionserver/c6f7b1897123:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-23T14:28:06,959 DEBUG [RS:0;c6f7b1897123:35385 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-23T14:28:06,983 INFO [RS_CLOSE_META-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/hbase/meta/1588230740/.tmp/info/5b5a8d2725b34605b2ff463dda1e6f42 2024-11-23T14:28:07,001 DEBUG [RS_CLOSE_META-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/hbase/meta/1588230740/.tmp/ns/db13ecc7dc8c4c07a1207129e9334834 is 93, key is TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe./ns:/1732371934900/DeleteFamily/seqid=0 2024-11-23T14:28:07,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742512_1688 (size=5906) 2024-11-23T14:28:07,159 DEBUG [RS:0;c6f7b1897123:35385 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-23T14:28:07,359 DEBUG [RS:0;c6f7b1897123:35385 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-23T14:28:07,405 INFO [RS_CLOSE_META-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/hbase/meta/1588230740/.tmp/ns/db13ecc7dc8c4c07a1207129e9334834 2024-11-23T14:28:07,427 DEBUG [RS_CLOSE_META-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/hbase/meta/1588230740/.tmp/rep_barrier/dd7663bda496433e87979a0aad05780c is 102, key is TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe./rep_barrier:/1732371934900/DeleteFamily/seqid=0 2024-11-23T14:28:07,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742513_1689 (size=6025) 2024-11-23T14:28:07,560 DEBUG [RS:0;c6f7b1897123:35385 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-23T14:28:07,760 DEBUG [RS:0;c6f7b1897123:35385 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-23T14:28:07,835 INFO [RS_CLOSE_META-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/hbase/meta/1588230740/.tmp/rep_barrier/dd7663bda496433e87979a0aad05780c 2024-11-23T14:28:07,858 DEBUG [RS_CLOSE_META-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/hbase/meta/1588230740/.tmp/table/5785c439425242ed98f25b230a6ba8c8 is 96, key is TestAcidGuarantees,,1732371907497.5d1382f36edf9c126fa17e1f99c7f3fe./table:/1732371934900/DeleteFamily/seqid=0 2024-11-23T14:28:07,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742514_1690 (size=5936) 2024-11-23T14:28:07,960 DEBUG [RS:0;c6f7b1897123:35385 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-23T14:28:08,160 DEBUG [RS:0;c6f7b1897123:35385 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-23T14:28:08,261 INFO [RS_CLOSE_META-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/hbase/meta/1588230740/.tmp/table/5785c439425242ed98f25b230a6ba8c8 2024-11-23T14:28:08,265 DEBUG [RS_CLOSE_META-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/hbase/meta/1588230740/.tmp/info/5b5a8d2725b34605b2ff463dda1e6f42 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/hbase/meta/1588230740/info/5b5a8d2725b34605b2ff463dda1e6f42 2024-11-23T14:28:08,268 INFO [RS_CLOSE_META-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/hbase/meta/1588230740/info/5b5a8d2725b34605b2ff463dda1e6f42, entries=12, sequenceid=90, filesize=6.4 K 2024-11-23T14:28:08,268 DEBUG [RS_CLOSE_META-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/hbase/meta/1588230740/.tmp/ns/db13ecc7dc8c4c07a1207129e9334834 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/hbase/meta/1588230740/ns/db13ecc7dc8c4c07a1207129e9334834 2024-11-23T14:28:08,271 INFO [RS_CLOSE_META-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/hbase/meta/1588230740/ns/db13ecc7dc8c4c07a1207129e9334834, entries=8, sequenceid=90, filesize=5.8 K 2024-11-23T14:28:08,272 DEBUG [RS_CLOSE_META-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/hbase/meta/1588230740/.tmp/rep_barrier/dd7663bda496433e87979a0aad05780c as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/hbase/meta/1588230740/rep_barrier/dd7663bda496433e87979a0aad05780c 2024-11-23T14:28:08,275 INFO [RS_CLOSE_META-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/hbase/meta/1588230740/rep_barrier/dd7663bda496433e87979a0aad05780c, entries=6, sequenceid=90, filesize=5.9 K 2024-11-23T14:28:08,275 DEBUG [RS_CLOSE_META-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/hbase/meta/1588230740/.tmp/table/5785c439425242ed98f25b230a6ba8c8 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/hbase/meta/1588230740/table/5785c439425242ed98f25b230a6ba8c8 2024-11-23T14:28:08,279 INFO [RS_CLOSE_META-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/hbase/meta/1588230740/table/5785c439425242ed98f25b230a6ba8c8, entries=7, sequenceid=90, filesize=5.8 K 2024-11-23T14:28:08,279 INFO [RS_CLOSE_META-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1720ms, sequenceid=90, compaction requested=false 2024-11-23T14:28:08,285 DEBUG [RS_CLOSE_META-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-23T14:28:08,286 DEBUG [RS_CLOSE_META-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-23T14:28:08,286 INFO [RS_CLOSE_META-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-23T14:28:08,286 DEBUG [RS_CLOSE_META-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732372086558Running coprocessor pre-close hooks at 1732372086558Disabling compacts and flushes for region at 1732372086558Disabling writes for close at 1732372086559 (+1 ms)Obtaining lock to block concurrent updates at 1732372086559Preparing flush snapshotting stores in 1588230740 at 1732372086559Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1732372086559Flushing stores of hbase:meta,,1.1588230740 at 1732372086559Flushing 1588230740/info: creating writer at 1732372086560 (+1 ms)Flushing 1588230740/info: appending metadata at 1732372086579 (+19 ms)Flushing 1588230740/info: closing flushed file at 1732372086579Flushing 1588230740/ns: creating writer at 1732372086987 (+408 ms)Flushing 1588230740/ns: appending metadata at 1732372087001 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1732372087001Flushing 1588230740/rep_barrier: creating writer at 1732372087412 (+411 ms)Flushing 1588230740/rep_barrier: appending metadata at 1732372087426 (+14 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1732372087427 (+1 ms)Flushing 1588230740/table: creating writer at 1732372087838 (+411 ms)Flushing 1588230740/table: appending metadata at 1732372087858 (+20 ms)Flushing 1588230740/table: closing flushed file at 1732372087858Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4b23ae84: reopening flushed file at 1732372088264 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@78086682: reopening flushed file at 1732372088268 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@53ecfa: reopening flushed file at 1732372088271 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@d965101: reopening flushed file at 1732372088275 (+4 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1720ms, sequenceid=90, compaction requested=false at 1732372088279 (+4 ms)Writing region close event to WAL at 1732372088283 (+4 ms)Running coprocessor post-close hooks at 1732372088285 (+2 ms)Closed at 1732372088286 (+1 ms) 2024-11-23T14:28:08,286 DEBUG [RS_CLOSE_META-regionserver/c6f7b1897123:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-23T14:28:08,360 INFO [RS:0;c6f7b1897123:35385 {}] regionserver.HRegionServer(976): stopping server c6f7b1897123,35385,1732371902767; all regions closed. 2024-11-23T14:28:08,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741834_1010 (size=25344) 2024-11-23T14:28:08,366 DEBUG [RS:0;c6f7b1897123:35385 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/oldWALs 2024-11-23T14:28:08,366 INFO [RS:0;c6f7b1897123:35385 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL c6f7b1897123%2C35385%2C1732371902767.meta:.meta(num 1732371906995) 2024-11-23T14:28:08,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741832_1008 (size=15630749) 2024-11-23T14:28:08,370 DEBUG [RS:0;c6f7b1897123:35385 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/oldWALs 2024-11-23T14:28:08,370 INFO [RS:0;c6f7b1897123:35385 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL c6f7b1897123%2C35385%2C1732371902767:(num 1732371905930) 2024-11-23T14:28:08,370 DEBUG [RS:0;c6f7b1897123:35385 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T14:28:08,370 INFO [RS:0;c6f7b1897123:35385 {}] regionserver.LeaseManager(133): Closed leases 2024-11-23T14:28:08,370 INFO [RS:0;c6f7b1897123:35385 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-23T14:28:08,370 INFO [RS:0;c6f7b1897123:35385 {}] hbase.ChoreService(370): Chore service for: regionserver/c6f7b1897123:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-23T14:28:08,370 INFO [RS:0;c6f7b1897123:35385 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-23T14:28:08,370 INFO [regionserver/c6f7b1897123:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-23T14:28:08,371 INFO [RS:0;c6f7b1897123:35385 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:35385 2024-11-23T14:28:08,418 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38947-0x101681565570000, quorum=127.0.0.1:50153, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-23T14:28:08,418 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35385-0x101681565570001, quorum=127.0.0.1:50153, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/c6f7b1897123,35385,1732371902767 2024-11-23T14:28:08,418 INFO [RS:0;c6f7b1897123:35385 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-23T14:28:08,429 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [c6f7b1897123,35385,1732371902767] 2024-11-23T14:28:08,439 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/c6f7b1897123,35385,1732371902767 already deleted, retry=false 2024-11-23T14:28:08,439 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; c6f7b1897123,35385,1732371902767 expired; onlineServers=0 2024-11-23T14:28:08,439 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'c6f7b1897123,38947,1732371901705' ***** 2024-11-23T14:28:08,439 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-23T14:28:08,439 INFO [M:0;c6f7b1897123:38947 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-23T14:28:08,439 INFO [M:0;c6f7b1897123:38947 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-23T14:28:08,440 DEBUG [M:0;c6f7b1897123:38947 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-23T14:28:08,440 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-23T14:28:08,440 DEBUG [M:0;c6f7b1897123:38947 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-23T14:28:08,440 DEBUG [master/c6f7b1897123:0:becomeActiveMaster-HFileCleaner.large.0-1732371905642 {}] cleaner.HFileCleaner(306): Exit Thread[master/c6f7b1897123:0:becomeActiveMaster-HFileCleaner.large.0-1732371905642,5,FailOnTimeoutGroup] 2024-11-23T14:28:08,440 DEBUG [master/c6f7b1897123:0:becomeActiveMaster-HFileCleaner.small.0-1732371905644 {}] cleaner.HFileCleaner(306): Exit Thread[master/c6f7b1897123:0:becomeActiveMaster-HFileCleaner.small.0-1732371905644,5,FailOnTimeoutGroup] 2024-11-23T14:28:08,440 INFO [M:0;c6f7b1897123:38947 {}] hbase.ChoreService(370): Chore service for: master/c6f7b1897123:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-23T14:28:08,440 INFO [M:0;c6f7b1897123:38947 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-23T14:28:08,441 DEBUG [M:0;c6f7b1897123:38947 {}] master.HMaster(1795): Stopping service threads 2024-11-23T14:28:08,441 INFO [M:0;c6f7b1897123:38947 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-23T14:28:08,441 INFO [M:0;c6f7b1897123:38947 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-23T14:28:08,441 ERROR [M:0;c6f7b1897123:38947 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-23T14:28:08,441 INFO [M:0;c6f7b1897123:38947 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-23T14:28:08,442 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-23T14:28:08,449 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38947-0x101681565570000, quorum=127.0.0.1:50153, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-23T14:28:08,449 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38947-0x101681565570000, quorum=127.0.0.1:50153, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T14:28:08,450 DEBUG [M:0;c6f7b1897123:38947 {}] zookeeper.ZKUtil(347): master:38947-0x101681565570000, quorum=127.0.0.1:50153, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-23T14:28:08,450 WARN [M:0;c6f7b1897123:38947 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-23T14:28:08,450 INFO [M:0;c6f7b1897123:38947 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/.lastflushedseqids 2024-11-23T14:28:08,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742515_1691 (size=72) 2024-11-23T14:28:08,529 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35385-0x101681565570001, quorum=127.0.0.1:50153, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T14:28:08,529 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35385-0x101681565570001, quorum=127.0.0.1:50153, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T14:28:08,529 INFO [RS:0;c6f7b1897123:35385 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-23T14:28:08,529 INFO [RS:0;c6f7b1897123:35385 {}] regionserver.HRegionServer(1031): Exiting; stopping=c6f7b1897123,35385,1732371902767; zookeeper connection closed. 2024-11-23T14:28:08,530 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2067e16f {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2067e16f 2024-11-23T14:28:08,530 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-23T14:28:08,861 INFO [M:0;c6f7b1897123:38947 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-23T14:28:08,861 INFO [M:0;c6f7b1897123:38947 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-23T14:28:08,861 DEBUG [M:0;c6f7b1897123:38947 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-23T14:28:08,861 INFO [M:0;c6f7b1897123:38947 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T14:28:08,861 DEBUG [M:0;c6f7b1897123:38947 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T14:28:08,861 DEBUG [M:0;c6f7b1897123:38947 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-23T14:28:08,861 DEBUG [M:0;c6f7b1897123:38947 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T14:28:08,861 INFO [M:0;c6f7b1897123:38947 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=773.87 KB heapSize=952.48 KB 2024-11-23T14:28:08,876 DEBUG [M:0;c6f7b1897123:38947 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/3ad1104ed8e440108f29dd2a7cb3d68b is 82, key is hbase:meta,,1/info:regioninfo/1732371907085/Put/seqid=0 2024-11-23T14:28:08,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742516_1692 (size=5672) 2024-11-23T14:28:09,279 INFO [M:0;c6f7b1897123:38947 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2206 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/3ad1104ed8e440108f29dd2a7cb3d68b 2024-11-23T14:28:09,305 DEBUG [M:0;c6f7b1897123:38947 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/63f3edfc80b2434fa0b9200139442a39 is 2276, key is \x00\x00\x00\x00\x00\x00\x00'/proc:d/1732371938125/Put/seqid=0 2024-11-23T14:28:09,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742517_1693 (size=45248) 2024-11-23T14:28:09,715 INFO [M:0;c6f7b1897123:38947 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=773.32 KB at sequenceid=2206 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/63f3edfc80b2434fa0b9200139442a39 2024-11-23T14:28:09,718 INFO [M:0;c6f7b1897123:38947 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 63f3edfc80b2434fa0b9200139442a39 2024-11-23T14:28:09,738 DEBUG [M:0;c6f7b1897123:38947 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/93e2f156627c40359ac14b894e234611 is 69, key is c6f7b1897123,35385,1732371902767/rs:state/1732371905680/Put/seqid=0 2024-11-23T14:28:09,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073742518_1694 (size=5156) 2024-11-23T14:28:10,142 INFO [M:0;c6f7b1897123:38947 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2206 (bloomFilter=true), to=hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/93e2f156627c40359ac14b894e234611 2024-11-23T14:28:10,145 DEBUG [M:0;c6f7b1897123:38947 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/3ad1104ed8e440108f29dd2a7cb3d68b as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/3ad1104ed8e440108f29dd2a7cb3d68b 2024-11-23T14:28:10,148 INFO [M:0;c6f7b1897123:38947 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/3ad1104ed8e440108f29dd2a7cb3d68b, entries=8, sequenceid=2206, filesize=5.5 K 2024-11-23T14:28:10,149 DEBUG [M:0;c6f7b1897123:38947 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/63f3edfc80b2434fa0b9200139442a39 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/63f3edfc80b2434fa0b9200139442a39 2024-11-23T14:28:10,151 INFO [M:0;c6f7b1897123:38947 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 63f3edfc80b2434fa0b9200139442a39 2024-11-23T14:28:10,151 INFO [M:0;c6f7b1897123:38947 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/63f3edfc80b2434fa0b9200139442a39, entries=176, sequenceid=2206, filesize=44.2 K 2024-11-23T14:28:10,152 DEBUG [M:0;c6f7b1897123:38947 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/93e2f156627c40359ac14b894e234611 as hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/93e2f156627c40359ac14b894e234611 2024-11-23T14:28:10,154 INFO [M:0;c6f7b1897123:38947 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43613/user/jenkins/test-data/97bce110-9b12-4ed0-8e00-33ae7a88a11a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/93e2f156627c40359ac14b894e234611, entries=1, sequenceid=2206, filesize=5.0 K 2024-11-23T14:28:10,154 INFO [M:0;c6f7b1897123:38947 {}] regionserver.HRegion(3140): Finished flush of dataSize ~773.87 KB/792446, heapSize ~952.18 KB/975032, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1293ms, sequenceid=2206, compaction requested=false 2024-11-23T14:28:10,155 INFO [M:0;c6f7b1897123:38947 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T14:28:10,155 DEBUG [M:0;c6f7b1897123:38947 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732372088861Disabling compacts and flushes for region at 1732372088861Disabling writes for close at 1732372088861Obtaining lock to block concurrent updates at 1732372088861Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732372088861Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=792446, getHeapSize=975272, getOffHeapSize=0, getCellsCount=2407 at 1732372088861Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732372088862 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732372088862Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732372088875 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732372088875Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732372089283 (+408 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732372089305 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732372089305Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732372089718 (+413 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732372089737 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732372089737Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@265f64bf: reopening flushed file at 1732372090145 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2c6da436: reopening flushed file at 1732372090148 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2c1085f2: reopening flushed file at 1732372090151 (+3 ms)Finished flush of dataSize ~773.87 KB/792446, heapSize ~952.18 KB/975032, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1293ms, sequenceid=2206, compaction requested=false at 1732372090154 (+3 ms)Writing region close event to WAL at 1732372090155 (+1 ms)Closed at 1732372090155 2024-11-23T14:28:10,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38519 is added to blk_1073741830_1006 (size=937059) 2024-11-23T14:28:10,158 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-23T14:28:10,158 INFO [M:0;c6f7b1897123:38947 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-23T14:28:10,158 INFO [M:0;c6f7b1897123:38947 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:38947 2024-11-23T14:28:10,158 INFO [M:0;c6f7b1897123:38947 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-23T14:28:10,318 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38947-0x101681565570000, quorum=127.0.0.1:50153, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T14:28:10,318 INFO [M:0;c6f7b1897123:38947 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-23T14:28:10,318 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38947-0x101681565570000, quorum=127.0.0.1:50153, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T14:28:10,323 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3dfb1097{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-23T14:28:10,325 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@16142ff{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-23T14:28:10,325 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-23T14:28:10,325 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5ef5f873{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-23T14:28:10,325 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4b8e8cdd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f90574b9-3de4-229a-1273-a648b15280c9/hadoop.log.dir/,STOPPED} 2024-11-23T14:28:10,328 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-23T14:28:10,328 WARN [BP-1838119457-172.17.0.3-1732371897891 heartbeating to localhost/127.0.0.1:43613 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-23T14:28:10,328 WARN [BP-1838119457-172.17.0.3-1732371897891 heartbeating to localhost/127.0.0.1:43613 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1838119457-172.17.0.3-1732371897891 (Datanode Uuid d2a26ef9-0122-4224-9c19-f41d6fffe1e6) service to localhost/127.0.0.1:43613 2024-11-23T14:28:10,328 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-23T14:28:10,330 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f90574b9-3de4-229a-1273-a648b15280c9/cluster_ea4d8525-e5f6-df4a-1eac-0a7a4e8ce8de/data/data1/current/BP-1838119457-172.17.0.3-1732371897891 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-23T14:28:10,330 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f90574b9-3de4-229a-1273-a648b15280c9/cluster_ea4d8525-e5f6-df4a-1eac-0a7a4e8ce8de/data/data2/current/BP-1838119457-172.17.0.3-1732371897891 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-23T14:28:10,331 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-23T14:28:10,339 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1bbf242f{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-23T14:28:10,339 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@34657887{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-23T14:28:10,339 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-23T14:28:10,340 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@41752735{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-23T14:28:10,340 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7def033e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f90574b9-3de4-229a-1273-a648b15280c9/hadoop.log.dir/,STOPPED} 2024-11-23T14:28:10,359 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-23T14:28:10,508 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down